{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ChatGLM3 implemented from scratch with kvcache\n",
    "in this file, i implemented chatglm3-6B from scratch, one tensor and matrix multiplication at a time.\n",
    "<br><br>\n",
    "the weights is huggingface format, suffix with safetensors.\n",
    "<br><br>\n",
    "use \"Genius is one percent inspiration and\" as prompt and generate result \"ninety-nine percent perspiration.\" in an auto-regressive manner.\n",
    "<br><br>\n",
    "Cache the vector of k and v during the prefill phase. and load it in decoder phase.\n",
    "\n",
    "从头开始实现ChatGLM3，支持huggingface格式，以safetensors为后缀的权重，并支持kvcache，进行自回归推理。\n",
    "\n",
    "以 \"Genius is one percent inspiration and\" 作为输入的prompt，先进行prefill的并行运算，生成并缓存 k 和 v，然后进行自回归生成，将前一次输出作为下一次输入，同时加载缓存的kvcache，进行推理，逐个token生成\"ninety-nine percent perspiration\"。\n",
    "\n",
    "前面的推理和原过程是一样的，只是在迭代layer生成第一个token时，缓存了k和v。生成第一个token之后，将其作为输入，是新增部分。\n",
    "\n",
    "这里以chatglm3-6BB为例，其他不同大小的模型需要修改个别参数适配。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 2 0\n",
      "['▁hello', '▁world', '!']\n",
      "[24954, 993, 30992]\n"
     ]
    }
   ],
   "source": [
    "from sentencepiece import SentencePieceProcessor\n",
    "import os\n",
    "model_path = '../ZhipuAI/chatglm3-6b-32k/'\n",
    "tokenizer_path = os.path.join(model_path, \"tokenizer.model\")\n",
    "sp_model = SentencePieceProcessor(model_file=tokenizer_path)\n",
    "bos_id = sp_model.bos_id()\n",
    "eos_id = sp_model.eos_id()\n",
    "pad_id = sp_model.unk_id()\n",
    "print(bos_id, eos_id, pad_id)\n",
    "\n",
    "tokens = sp_model.EncodeAsPieces(\"hello world!\")\n",
    "print(tokens)\n",
    "\n",
    "tokens = sp_model.encode(\"hello world!\")\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## reading the model file\n",
    "normally, reading this depends on how the model classes are written and the variable names inside them.\n",
    "<br>\n",
    "but since we are implementing qwen1.5 from scratch we will read the file one tensor at a time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformer.embedding.word_embeddings.weight torch.Size([65024, 4096])\n",
      "transformer.rotary_pos_emb.inv_freq torch.Size([32])\n",
      "transformer.encoder.layers.0.input_layernorm.weight torch.Size([4096])\n",
      "transformer.encoder.layers.0.self_attention.query_key_value.weight torch.Size([4608, 4096])\n",
      "transformer.encoder.layers.0.self_attention.query_key_value.bias torch.Size([4608])\n",
      "transformer.encoder.layers.0.self_attention.dense.weight torch.Size([4096, 4096])\n",
      "transformer.encoder.layers.0.post_attention_layernorm.weight torch.Size([4096])\n",
      "transformer.encoder.layers.0.mlp.dense_h_to_4h.weight torch.Size([27392, 4096])\n",
      "transformer.encoder.layers.0.mlp.dense_4h_to_h.weight torch.Size([4096, 13696])\n",
      "transformer.encoder.layers.1.input_layernorm.weight torch.Size([4096])\n",
      "transformer.encoder.layers.1.self_attention.query_key_value.weight torch.Size([4608, 4096])\n",
      "transformer.encoder.layers.1.self_attention.query_key_value.bias torch.Size([4608])\n",
      "transformer.encoder.layers.1.self_attention.dense.weight torch.Size([4096, 4096])\n",
      "transformer.encoder.layers.1.post_attention_layernorm.weight torch.Size([4096])\n",
      "transformer.encoder.layers.1.mlp.dense_h_to_4h.weight torch.Size([27392, 4096])\n",
      "transformer.encoder.layers.1.mlp.dense_4h_to_h.weight torch.Size([4096, 13696])\n",
      "transformer.encoder.layers.2.input_layernorm.weight torch.Size([4096])\n",
      "transformer.encoder.layers.2.self_attention.query_key_value.weight torch.Size([4608, 4096])\n",
      "transformer.encoder.layers.2.self_attention.query_key_value.bias torch.Size([4608])\n",
      "transformer.encoder.layers.2.self_attention.dense.weight torch.Size([4096, 4096])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import json\n",
    "model = {}\n",
    "for i in range(1,8):  # 遍历加载权重文件，不同大小的模型需要针对性修改\n",
    "    tmp_model = torch.load(os.path.join(model_path, f\"pytorch_model-0000{i}-of-00007.bin\"), map_location=torch.device('cpu'))\n",
    "    model.update(tmp_model)\n",
    "    \n",
    "#print(json.dumps(list(model.keys())[:20], indent=4))\n",
    "for k in list(model.keys())[:20]:\n",
    "    print(k, model[k].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'add_bias_linear': False,\n",
       " 'add_qkv_bias': True,\n",
       " 'apply_query_key_layer_scaling': True,\n",
       " 'apply_residual_connection_post_layernorm': False,\n",
       " 'attention_dropout': 0.0,\n",
       " 'attention_softmax_in_fp32': True,\n",
       " 'bias_dropout_fusion': True,\n",
       " 'ffn_hidden_size': 13696,\n",
       " 'fp32_residual_connection': False,\n",
       " 'hidden_dropout': 0.0,\n",
       " 'hidden_size': 4096,\n",
       " 'kv_channels': 128,\n",
       " 'layernorm_epsilon': 1e-05,\n",
       " 'rope_ratio': 50,\n",
       " 'multi_query_attention': True,\n",
       " 'multi_query_group_num': 2,\n",
       " 'num_attention_heads': 32,\n",
       " 'num_layers': 28,\n",
       " 'original_rope': True,\n",
       " 'padded_vocab_size': 65024,\n",
       " 'post_layer_norm': True,\n",
       " 'rmsnorm': True,\n",
       " 'seq_length': 32768,\n",
       " 'use_cache': True,\n",
       " 'torch_dtype': 'float16',\n",
       " 'transformers_version': '4.30.2',\n",
       " 'tie_word_embeddings': False,\n",
       " 'eos_token_id': 2,\n",
       " 'pad_token_id': 0}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(os.path.join(model_path, 'config.json'), \"r\") as f:\n",
    "    config = json.load(f)\n",
    "dict(zip(list(config.keys())[4:], list(config.values())[4:]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we use this config to infer details about the model like\n",
    "1. the model has 40 transformer layers\n",
    "2. each multi-head attention block has 20 heads\n",
    "3. the vocab size and so on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim = config[\"hidden_size\"]\n",
    "n_layers = config[\"num_layers\"]\n",
    "n_heads = config[\"num_attention_heads\"]\n",
    "n_kv_heads = config[\"multi_query_group_num\"]\n",
    "n_kv_channels = config[\"kv_channels\"]\n",
    "vocab_size = config[\"padded_vocab_size\"]\n",
    "# multiple_of = config[\"multiple_of\"]\n",
    "ffn_dim_multiplier = config[\"ffn_hidden_size\"]\n",
    "norm_eps = config[\"layernorm_epsilon\"]\n",
    "rope_theta = torch.tensor(config[\"rope_ratio\"]) * 10000  # base = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## converting text to tokens\n",
    "here we use tiktoken (i think an openai library) as the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁Gen', 'ius', '▁is', '▁one', '▁percent', '▁inspiration', '▁and']\n",
      "[1, 5292, 3551, 323, 544, 2098, 8969, 293]\n",
      "['', 'Gen', 'ius', 'is', 'one', 'percent', 'inspiration', 'and']\n"
     ]
    }
   ],
   "source": [
    "# prompt = \"the answer to the ultimate question of life, the universe, and everything is \"\n",
    "# prompt = \"Failure is the mother of\"\n",
    "prompt = \"Genius is one percent inspiration and\"   # generate “ninety-nine percent perspiration. Thomas Edison”\n",
    "tokens = sp_model.EncodeAsPieces(prompt)\n",
    "print(tokens)\n",
    "\n",
    "tokens = [bos_id] + sp_model.encode(prompt)\n",
    "print(tokens)\n",
    "\n",
    "tokens = torch.tensor(tokens)\n",
    "prompt_split_as_tokens = [sp_model.decode([token.item()]) for token in tokens]\n",
    "print(prompt_split_as_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## converting tokens to their embedding\n",
    "IM SORRY but this is the only part of the codebase where i use an inbuilt neural network module\n",
    "<br>\n",
    "anyway, so our [7x1] tokens are now [2560], i.e. 7 embeddings (one for each token) of length 2560\n",
    "<br>\n",
    "<br>\n",
    "note: keep track of the shapes, it makes it much easier to understand everything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_layer = torch.nn.Embedding(vocab_size, dim)\n",
    "embedding_layer.weight.data.copy_(model[\"transformer.embedding.word_embeddings.weight\"])\n",
    "token_embeddings_unnormalized = embedding_layer(tokens).to(torch.bfloat16)\n",
    "token_embeddings_unnormalized.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we then normalize the embedding using rms normalization\n",
    "please, note after this step the shapes dont change, the values are just normalized\n",
    "<br>\n",
    "things to keep in mind, we need a norm_eps (from config) because we dont want to accidently set rms to 0 and divide by 0\n",
    "<br>\n",
    "here is the formula:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def rms_norm(tensor, norm_weights):\n",
    "#     rms = (tensor.pow(2).mean(-1, keepdim=True) + norm_eps)**0.5\n",
    "#     return tensor * (norm_weights / rms)\n",
    "def rms_norm(tensor, norm_weights):\n",
    "    return (tensor * torch.rsqrt(tensor.pow(2).mean(-1, keepdim=True) + norm_eps)) * norm_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# building the first layer of the transformer\n",
    "\n",
    "### normalization\n",
    "you will see me accessing layer.0 from the model dict (this is the first layer)\n",
    "<br>\n",
    "anyway, so after normalizing our shapes are still [7x2560] same as embedding but normalized "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_embeddings = rms_norm(token_embeddings_unnormalized, model[\"transformer.encoder.layers.0.input_layernorm.weight\"])\n",
    "token_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### attention implemented from scratch\n",
    "let's load the attention heads of the first layer of the transformer\n",
    "\n",
    "&gt; when we load the query, key, value and output vectors from the model we notice the shapes to be [2560x2560], [2560x2560], [2560x2560], [2560x2560]\n",
    "<br>\n",
    "&gt; at first glance this is weird because ideally we want each q,k,v and o for each head individually\n",
    "<br>\n",
    "&gt; the authors of the code bundled them togeather because its easy it helps parallize attention head multiplication.\n",
    "<br>\n",
    "&gt; im going to unwrap everything... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4608, 4096]) torch.Size([4608]) torch.Size([4096, 4096])\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "    model[\"transformer.encoder.layers.0.self_attention.query_key_value.weight\"].shape,\n",
    "    model[\"transformer.encoder.layers.0.self_attention.query_key_value.bias\"].shape,\n",
    "    model[\"transformer.encoder.layers.0.self_attention.dense.weight\"].shape,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### unwrapping query\n",
    "in the next section we will unwrap the queries from multiple attention heads, the resulting shape is [20x128x2560]\n",
    "<br><br>\n",
    "here, 20 is the number of attention heads in qwen1.5-4B, 128 is the size of the query vector and 2560 is the size of the token embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4096, 4096]) torch.Size([256, 4096]) torch.Size([256, 4096])\n",
      "torch.Size([32, 128, 4096])\n",
      "torch.Size([32, 128])\n"
     ]
    }
   ],
   "source": [
    "q_k_v_layer0 = model[\"transformer.encoder.layers.0.self_attention.query_key_value.weight\"]\n",
    "head_dim = q_k_v_layer0.shape[1] // n_heads\n",
    "q_layer0, k_layer0, v_layer0 = q_k_v_layer0.split([ \n",
    "                                                    n_heads * n_kv_channels,\n",
    "                                                    n_kv_heads * n_kv_channels,\n",
    "                                                    n_kv_heads * n_kv_channels,\n",
    "                                                  ], dim=0)\n",
    "print(q_layer0.shape, k_layer0.shape, v_layer0.shape)\n",
    "\n",
    "q_layer0 = q_layer0.reshape(n_heads, n_kv_channels, -1)\n",
    "print(q_layer0.shape)\n",
    "\n",
    "q_k_v_layer0_bias = model[\"transformer.encoder.layers.0.self_attention.query_key_value.bias\"]\n",
    "q_layer0_bias, k_layer0_bias, v_layer0_bias = q_k_v_layer0_bias.split([\n",
    "                                                                       n_heads * n_kv_channels,\n",
    "                                                                       n_kv_heads * n_kv_channels,\n",
    "                                                                       n_kv_heads * n_kv_channels,\n",
    "                                                                      ])\n",
    "q_layer0_bias = q_layer0_bias.reshape(n_heads, -1)\n",
    "print(q_layer0_bias.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### im going to implement the first head of the first layer\n",
    "here i access the query weight matrix first head of the first layer, the size of this query weight matrix is [128x2560]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 4096])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_layer0_head0 = q_layer0[0]\n",
    "q_layer0_head0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_layer0_bias_head0 = q_layer0_bias[0]\n",
    "q_layer0_bias_head0.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### we now multiply the query weights with the token embedding, to recive a query for the token\n",
    "here you can see the resulting shape is [7x128], this is because we have 7 tokens and for each token there is a 128 length query."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token = torch.matmul(token_embeddings, q_layer0_head0.T.float()) + q_layer0_bias_head0.float()\n",
    "q_per_token.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## positioning encoding\n",
    "we are now at a stage where we have a query vector for each token in our prompt, but if you think about it -- the indivitually query vector has no idea about the position in the prompt.\n",
    "<br><br>\n",
    "query: \"the answer to the ultimate question of life, the universe, and everything is \"\n",
    "<br><br>\n",
    "in our prompt we have used \"the\" three times, we need the query vectors of all 3 \"the\" tokens to have different query vectors (each of size [1x128]) based on their positions in the query. we perform these rotations using RoPE (rotory positional embedding).\n",
    "<br><br>\n",
    "### RoPE\n",
    "watch this video (this is what i watched) to understand the math.\n",
    "https://www.youtube.com/watch?v=o29P0Kpobz0&t=530s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64, 2])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)\n",
    "q_per_token_split_into_pairs.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in the above step, we split the query vectors into pairs, we apply a rotational angle shift to each pair!\n",
    "<br><br>\n",
    "we now have a vector of size [7x64x2], this is the 128 length queries split into 64 pairs for each token in the prompt! each of those 64 pairs will be rotated by m*(theta) where m is the position of the token for which we are rotating the query!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## using dot product of complex numbers to rotate a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0000, 0.0156, 0.0312, 0.0469, 0.0625, 0.0781, 0.0938, 0.1094, 0.1250,\n",
       "        0.1406, 0.1562, 0.1719, 0.1875, 0.2031, 0.2188, 0.2344, 0.2500, 0.2656,\n",
       "        0.2812, 0.2969, 0.3125, 0.3281, 0.3438, 0.3594, 0.3750, 0.3906, 0.4062,\n",
       "        0.4219, 0.4375, 0.4531, 0.4688, 0.4844, 0.5000, 0.5156, 0.5312, 0.5469,\n",
       "        0.5625, 0.5781, 0.5938, 0.6094, 0.6250, 0.6406, 0.6562, 0.6719, 0.6875,\n",
       "        0.7031, 0.7188, 0.7344, 0.7500, 0.7656, 0.7812, 0.7969, 0.8125, 0.8281,\n",
       "        0.8438, 0.8594, 0.8750, 0.8906, 0.9062, 0.9219, 0.9375, 0.9531, 0.9688,\n",
       "        0.9844])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zero_to_one_split_into_64_parts = torch.tensor(range(64))/64\n",
    "zero_to_one_split_into_64_parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1.0000e+00, 8.1462e-01, 6.6360e-01, 5.4058e-01, 4.4037e-01, 3.5873e-01,\n",
       "        2.9223e-01, 2.3805e-01, 1.9392e-01, 1.5797e-01, 1.2869e-01, 1.0483e-01,\n",
       "        8.5397e-02, 6.9566e-02, 5.6670e-02, 4.6164e-02, 3.7606e-02, 3.0635e-02,\n",
       "        2.4955e-02, 2.0329e-02, 1.6560e-02, 1.3490e-02, 1.0990e-02, 8.9523e-03,\n",
       "        7.2927e-03, 5.9407e-03, 4.8394e-03, 3.9423e-03, 3.2114e-03, 2.6161e-03,\n",
       "        2.1311e-03, 1.7360e-03, 1.4142e-03, 1.1520e-03, 9.3847e-04, 7.6450e-04,\n",
       "        6.2277e-04, 5.0732e-04, 4.1327e-04, 3.3666e-04, 2.7425e-04, 2.2341e-04,\n",
       "        1.8199e-04, 1.4825e-04, 1.2077e-04, 9.8381e-05, 8.0143e-05, 6.5286e-05,\n",
       "        5.3183e-05, 4.3324e-05, 3.5292e-05, 2.8750e-05, 2.3420e-05, 1.9078e-05,\n",
       "        1.5542e-05, 1.2660e-05, 1.0313e-05, 8.4015e-06, 6.8440e-06, 5.5752e-06,\n",
       "        4.5417e-06, 3.6997e-06, 3.0139e-06, 2.4551e-06])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "freqs = 1.0 / (rope_theta ** zero_to_one_split_into_64_parts)\n",
    "freqs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "abc = torch.arange(len(tokens))\n",
    "abc.shape\n",
    "idx_theta = torch.outer(abc, abc).float()\n",
    "cache = torch.stack([torch.cos(idx_theta), torch.sin(idx_theta)], dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 64])\n",
      "torch.Size([8, 64])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAACO8ElEQVR4nO3dd1jTVxcH8G9AhqDgQMG998Jq3aNWXFXrqhJtK1qrdVdxVK2j1lbUutoquMfb1oJbW/dstWpV4t6zWhXcoKgg5L5/nAaIrCQkub8k5/M8PLQh4wSQnNx77jkqIYQAY4wxxpidcJIdAGOMMcaYOXFywxhjjDG7wskNY4wxxuwKJzeMMcYYsyuc3DDGGGPMrnBywxhjjDG7wskNY4wxxuwKJzeMMcYYsyuc3DDGGGPMrnByw1g27d+/HyqVCvv375cdip6ffvoJFStWhIuLC/LkySM7HIfw3XffoXTp0nB2doa/v3+m13WUn49S/30w+8bJDWMZWLFiBVQqVfKHu7s7ypcvj8GDByM6Otosj7F161Z89dVXZrmv1C5evIhevXqhTJkyWLx4MRYtWmT2x2D6du7cidGjR6Nhw4ZYvnw5pk6dmuF1+efDmGXlkB0AY0r39ddfo1SpUnj16hUOHjyIsLAwbN26FWfPnoWHh0e27nvr1q2YP3++2ROc/fv3Q6vV4vvvv0fZsmXNet8sfXv37oWTkxOWLl0KV1fXTK/rSD+fJk2a4OXLl1l+TxgzJ05uGMtCmzZtULt2bQDAp59+ivz582P27NnYtGkTunfvLjm69N2/fx8A7Ga7Iy4uDp6enrLDyNT9+/eRM2dOg17EDf35CCHw6tUr5MyZ0xwhSuHk5AR3d3fZYTAHw9tSjBnp3XffBQDcuHEj0+utWbMGtWrVQs6cOeHj44OPPvoId+7cSf56r169MH/+fADQ2/7KSmhoKKpUqQI3NzcULlwYgwYNwtOnT5O/XrJkSUyaNAkAUKBAAahUqixXhvbu3YvGjRvD09MTefLkQYcOHXDhwgW963z11VdQqVS4evUqevXqhTx58sDb2xu9e/fGixcv0tznzz//nPz88+XLB7Vajdu3b2f5/HSPc/78efTo0QN58+ZFo0aNAACJiYmYMmUKypQpAzc3N5QsWRLjxo1DfHx88u2Dg4ORP39+CCGSLxsyZAhUKhV++OGH5Muio6OhUqkQFhaWaTyGPKZKpcLy5csRFxeX/HNcsWJFuveX2c+nZMmSaNeuHXbs2IHatWsjZ86cWLhwIQDg6dOnGDZsGIoVKwY3NzeULVsW06dPh1ar1bv/p0+folevXvD29kaePHkQFBSEkydPpokpKioKvXv3RtGiReHm5oZChQqhQ4cOuHnzZqbfjzfduXMHffr0QeHCheHm5oZSpUphwIABSEhIAJB+zc2VK1fQpUsX+Pn5wd3dHUWLFoVarUZMTIxRj81YRnjlhjEjXbt2DQCQP3/+DK+zYsUK9O7dG2+//TZCQkIQHR2N77//Hn/99RdOnDiBPHny4LPPPsPdu3exa9cu/PTTTwY99ldffYXJkycjICAAAwYMwKVLlxAWFoZjx47hr7/+gouLC+bOnYv//e9/2LBhA8LCwpArVy5Ur149w/vcvXs32rRpg9KlS+Orr77Cy5cv8eOPP6Jhw4bQaDQoWbKk3vW7deuGUqVKISQkBBqNBkuWLEHBggUxffr05Ot8++23mDBhArp164ZPP/0UDx48wI8//ogmTZokP/+sdO3aFeXKlcPUqVOTE5VPP/0UK1euxAcffIARI0bg77//RkhICC5cuIANGzYAABo3bow5c+bg3LlzqFq1KgDgwIEDcHJywoEDBzB06NDkywDaNsmMIY/5008/YdGiRTh69CiWLFkCAGjQoEG695fVz+fSpUvo3r07PvvsM/Tt2xcVKlTAixcv0LRpU9y5cwefffYZihcvjkOHDmHs2LG4d+8e5s6dC4BWejp06ICDBw+if//+qFSpEjZs2ICgoKA0cXTp0gXnzp3DkCFDULJkSdy/fx+7du3CrVu30vzMM3L37l3UqVMHT58+Rb9+/VCxYkXcuXMHa9euxYsXL9JdxUpISECrVq0QHx+PIUOGwM/PD3fu3MHvv/+Op0+fwtvb26DHZixTgjGWruXLlwsAYvfu3eLBgwfi9u3bIjw8XOTPn1/kzJlT/Pvvv0IIIfbt2ycAiH379gkhhEhISBAFCxYUVatWFS9fvky+v99//10AEBMnTky+bNCgQcLQf4b3798Xrq6uomXLliIpKSn58nnz5gkAYtmyZcmXTZo0SQAQDx48yPJ+/f39RcGCBcWjR4+SLzt16pRwcnISPXv2THOfn3zyid7tO3XqJPLnz5/8/zdv3hTOzs7i22+/1bvemTNnRI4cOdJc/ibd43Tv3l3v8pMnTwoA4tNPP9W7fOTIkQKA2Lt3rxCCvk8ARGhoqBBCiKdPnwonJyfRtWtX4evrm3y7oUOHinz58gmtVpthLIY+phBCBAUFCU9Pz0yf25vP8c2fT4kSJQQAsX37dr3Lp0yZIjw9PcXly5f1Lh8zZoxwdnYWt27dEkIIsXHjRgFAzJgxI/k6iYmJonHjxgKAWL58uRBCiCdPnggA4rvvvjMo3oz07NlTODk5iWPHjqX5mu77+ua/jxMnTggAYs2aNdl6bMYyw9tSjGUhICAABQoUQLFixaBWq5ErVy5s2LABRYoUSff6x48fx/379zFw4EC9WoO2bduiYsWK2LJli0lx7N69GwkJCRg2bBicnFL+6fbt2xdeXl4m3e+9e/dw8uRJ9OrVC/ny5Uu+vHr16mjRogW2bt2a5jb9+/fX+//GjRvj0aNHiI2NBQCsX78eWq0W3bp1w8OHD5M//Pz8UK5cOezbt8+g2N58HF0swcHBepePGDECAJKff4ECBVCxYkX8+eefAIC//voLzs7OGDVqFKKjo3HlyhUAtHLTqFGjTLcCDX1McypVqhRatWqld9maNWvQuHFj5M2bV+97GhAQgKSkpOTnunXrVuTIkQMDBgxIvq2zszOGDBmid3+62qD9+/fjyZMnJsWp1WqxceNGtG/fPrkmLbWMvq+6lZkdO3aku53JmDnwthRjWZg/fz7Kly+PHDlywNfXFxUqVNBLLt70zz//AAAqVKiQ5msVK1bEwYMHTYojo/t1dXVF6dKlk79ujvsEgEqVKmHHjh1pinmLFy+ud728efMCAJ48eQIvLy9cuXIFQgiUK1cu3cd0cXExKLZSpUqlidXJySnN6SI/Pz/kyZNH7/k3btw4OTE5cOAAateujdq1ayNfvnw4cOAAfH19cerUKfTo0SPTGIx5THN583kDVKNy+vRpFChQIN3b6AqU//nnHxQqVAi5cuXS+/qbP183NzdMnz4dI0aMgK+vL+rVq4d27dqhZ8+e8PPzMyjOBw8eIDY2Nnnrz1ClSpVCcHAwZs+ejV9++QWNGzfG+++/j48++oi3pJjZcHLDWBbq1KmT7jtTR+Xs7Jzu5eK/uhitVguVSoVt27ale903X3gzktEJIUOKrhs1aoTFixfj+vXrOHDgABo3bgyVSoVGjRrhwIEDKFy4MLRaLRo3bmxQLIY8prmk97y1Wi1atGiB0aNHp3ub8uXLG/04w4YNQ/v27bFx40bs2LEDEyZMQEhICPbu3YuaNWsafX/GmDVrFnr16oVNmzZh586dGDp0KEJCQnDkyBEULVrUoo/NHAMnN4yZWYkSJQBQYajuZJXOpUuXkr8OGPeimfp+S5cunXx5QkICbty4gYCAgGzF+qaLFy/Cx8fH6CPYZcqUgRACpUqVMulFNyMlSpSAVqvFlStXUKlSpeTLo6Oj8fTpU73vqy5p2bVrF44dO4YxY8YAoOLhsLAwFC5cGJ6enqhVq5bZHtOSypQpg+fPn2f5My5RogT27NmD58+f6yWR6f18dfc7YsQIjBgxAleuXIG/vz9mzZqFn3/+OcuYChQoAC8vL5w9e9a4J/OfatWqoVq1ahg/fjwOHTqEhg0bYsGCBfjmm29Muj/GUuOaG8bMrHbt2ihYsCAWLFigd1x427ZtuHDhAtq2bZt8mS5xSH2UOyMBAQFwdXXFDz/8oHfMeenSpYiJidG7X0MVKlQI/v7+WLlypV4MZ8+exc6dO/Hee+8ZfZ+dO3eGs7MzJk+erBcnQKs7jx49Mvo+ASTHojsZpDN79mwA0Hv+pUqVQpEiRTBnzhy8fv0aDRs2BEBJz7Vr17B27VrUq1cPOXJk/v7OmMe0pG7duuHw4cPYsWNHmq89ffoUiYmJACjexMREvePtSUlJ+PHHH/Vu8+LFC7x69UrvsjJlyiB37tx6v7OZcXJyQseOHfHbb7/h+PHjab7+5s9eJzY2NjlenWrVqsHJycngx2YsK7xyw5iZubi4YPr06ejduzeaNm2K7t27Jx8FL1myJIYPH558Xd3KwdChQ9GqVSs4OztDrVane78FChTA2LFjMXnyZLRu3Rrvv/8+Ll26hNDQULz99tv46KOPTIr3u+++Q5s2bVC/fn306dMn+Si4t7e3SZ2Ty5Qpg2+++QZjx47FzZs30bFjR+TOnRs3btzAhg0b0K9fP4wcOdLo+61RowaCgoKwaNEiPH36FE2bNsXRo0excuVKdOzYEc2aNdO7fuPGjREeHo5q1aol1wW99dZb8PT0xOXLl7OstzHlMS1l1KhR2Lx5M9q1a4devXqhVq1aiIuLw5kzZ7B27VrcvHkTPj4+aN++PRo2bIgxY8bg5s2bqFy5MtavX5+mf8zly5fRvHlzdOvWDZUrV0aOHDmwYcMGREdHZ/j7l56pU6di586daNq0Kfr164dKlSrh3r17WLNmDQ4ePJjukf+9e/di8ODB6Nq1K8qXL4/ExET89NNPcHZ2RpcuXbL7rWKMSDypxZii6Y6Cp3fMNbU3j7rqREREiJo1awo3NzeRL18+8eGHHyYfH9dJTEwUQ4YMEQUKFBAqlcqgY+Hz5s0TFStWFC4uLsLX11cMGDBAPHnyRO86xhwFF0KI3bt3i4YNG4qcOXMKLy8v0b59e3H+/HmD7lP3fbpx44be5evWrRONGjUSnp6ewtPTU1SsWFEMGjRIXLp0KdNYMov99evXYvLkyaJUqVLCxcVFFCtWTIwdO1a8evUqzXXnz58vAIgBAwboXR4QECAAiD179mQah7GPaa6j4G3btk33Ns+ePRNjx44VZcuWFa6ursLHx0c0aNBAzJw5UyQkJCRf79GjR+Ljjz8WXl5ewtvbW3z88cfJx691R8EfPnwoBg0aJCpWrCg8PT2Ft7e3qFu3rli9erVB8af2zz//iJ49e4oCBQoINzc3Ubp0aTFo0CARHx8vhEj77+P69evik08+EWXKlBHu7u4iX758olmzZmL37t1GPzZjGVEJkcHaIWOMMbtw8+ZNlCpVCsuXL0evXr1kh8OYxXHNDWOMMcbsCtfcMMYY0/P8+XM8f/480+sUKFAgw7YAjMnGyQ1jjDE9M2fOxOTJkzO9zo0bNwyeQcWYtXHNDWOMMT3Xr1/H9evXM71Oo0aN9MaLMKYknNwwxhhjzK5wQTFjjDHG7IrD1dxotVrcvXsXuXPntuq8GMYYY4yZTgiBZ8+eoXDhwpkOLwYcMLm5e/cuihUrJjsMxhhjjJng9u3bWQ5YdbjkJnfu3ADom+Pl5SU5GsYYY4wZIjY2FsWKFUt+Hc+MwyU3uq0oLy8vTm4YY4wxG2NISQkXFDPGGGPMrnBywxhjjDG7wskNY4wxxuwKJzeMMcYYsyuc3DDGWHrCwoDq1QEvL/qoXx/Ytk12VIwxA3Bywxhj6SlaFJg2DYiMBI4fB959F+jQATh3TnZkjLEsONxRcMYYM0j79vr//+23tJpz5AhQpYqcmBhjBuHkhjHGspKUBKxZA8TF0fYUY0zROLlhjLGMnDlDycyrV0CuXMCGDUDlyrKjYoxlQWrNzZ9//on27dujcOHCUKlU2LhxY5a32b9/P9566y24ubmhbNmyWLFihcXjZIw5qAoVgJMngb//BgYMAIKCgPPnZUfFGMuC1OQmLi4ONWrUwPz58w26/o0bN9C2bVs0a9YMJ0+exLBhw/Dpp59ix44dFo6UMeaQXF2BsmWBWrWAkBCgRg3g++9lR8UYy4LUbak2bdqgTZs2Bl9/wYIFKFWqFGbNmgUAqFSpEg4ePIg5c+agVatWlgqTMcaIVgvEx8uOgjGWBZuquTl8+DACAgL0LmvVqhWGDRuW4W3i4+MRn+qPUWxsrKXCY4zZk7FjgTZtgOLFgWfPgFWrgP37AV4pZkzxbKrPTVRUFHx9ffUu8/X1RWxsLF6+fJnubUJCQuDt7Z38UaxYMWuEyhizUX/+CdSrB8T/ex/o2ZPqbpo3B44do8SmRQvZITLGsmBTyY0pxo4di5iYmOSP27dvyw6JMaZgcXFUP/wgZClw8yZtQ92/D+zezYmNsaZNA1QqIJPVdcYswaa2pfz8/BAdHa13WXR0NLy8vJAzZ850b+Pm5gY3NzdrhMcYswMeHvT5xQu5cdi8Y8eAhQtphAVjVmZTKzf169fHnj179C7btWsX6nNTLcaYmXByYwbPnwMffggsXgzkzSs7GuaApCY3z58/x8mTJ3Hy5EkAdNT75MmTuHXrFgDaUurZs2fy9fv374/r169j9OjRuHjxIkJDQ7F69WoMHz5cRviMMTvEyY0ZDBoEtG0LvHEAhDFrkbotdfz4cTRr1iz5/4ODgwEAQUFBWLFiBe7du5ec6ABAqVKlsGXLFgwfPhzff/89ihYtiiVLlvAxcMaY2XByk03h4YBGQ9tSjEkiNbl55513IITI8OvpdR9+5513cOLECQtGxRhzZJzcZMPt28DnnwO7dgHu7rKjYQ7MpgqKGWPM0ji5yYbISDpZ9tZbKZclJdH5+nnz6OSZs7O8+JjD4OSGMcZS0R285OTGBM2b07DR1Hr3BipWBL74ghMbZjWc3DDGWCo5ctBIKU5uTJA7N1C1qv5lnp5A/vxpL2fMgmzqKDhjjFmDhwcnN8b46y+gUSPakWJMCXjlhjHG3sDJjXH+/psOSOXPn84X9++3djiM8coNY4y9iZMb40RGAv7+XFLDlIOTG8YYewMnN8bRaPQPSDEmGyc3jDH2Bk5uDPf8OXDpEic3TFk4uWGMsTdwcmO4U6cAIYBatWRHwlgKTm4YY+wNnNwYTqOho/OVK8uOhLEUnNwwxtgbOLkxnEYDVK8OuLjIjoSxFJzcMMbYGzi5MRwXEzMl4uSGMcbewMmNYV69As6d4+SGKQ8nN4zZipAQ4O23qcV9wYJAx450TIWZHSc3hjl9muZicnLDlIaTG8ZsxR9/AIMGAUeOALt2Aa9fAy1bAnFxsiOzO5zcGEajocZ91arJjoQxfTx+gTFbsX27/v+vWEErOJGRQJMmUkKyV5zcGEajAapUAdzdZUfCmD5euWHMVsXE0Od8+eTGYYc8PTm5MYRGw/1tmDJxcsOYLdJqgWHDgIYNgapVZUdjdzw8gPh4qidh6UtIAM6c4Xobpky8LcWYLRo0CDh7Fjh4UHYkdsnDgz6/fAnkyiU3FqU6f54SHE5umBLxyg1jtmbwYOD334F9+4CiRWVHY5d0yQ1vTWVMowFUKqBGDdmRMJYWr9wwZiuEAIYMATZsAPbvB0qVkh2R3eLkJmsaDVCxItUnMaY0nNwwZgPi4gDPUYOAVauATZuo101UFH3R2xvImVNugHaGk5usRUbylhRTLt6WYkzhpk+n2T0IC6MTUu+8AxQqlPIRESE7RLvDyU3mEhNpGjgnN0ypOLlhTOHefhu4fh34+4igrak3P3r1kh2i3eHkJnOXLlGxNSc3TKk4uWFM4Zo2BXx9gfBw2ZE4Dk5uMqfR0OeaNeXGIdVXX1FFdeqPihVlR8X+w8kNYwrn7Ax06wasXk3tbZjlcXKTOY0GKFuWyr0cWpUqwL17KR/cmkExOLlhzAYEBgJ37/LfTmvh5CZzGg1vSQEAcuQA/PxSPnx8ZEfE/sPJDWM2oH59oFgx3pqyFjc32mXg5CYtrRY4cYKTGwDAlStA4cJA6dLAhx8Ct27Jjoj9h5MbxmyAkxOt3qxZQydVmGWpVDw8MyPXrgHPnnFyg7p1aXjt9u10kvHGDaBxY/rmMOk4uWHMRqjVwMOHwN69siNxDJzcpC8ykj47dDExALRpA3TtSn0aWrUCtm4Fnj6l4jgmHSc3jNmIt94CypThtjbWwslN+jQaoHhxLi9JI08eoHx54OpV2ZEwcHLDmM1QqWj1Zv16mljNLIuTm/RxMXEGnj+nPbtChWRHwsDJDWM2Ra2mle+dO2VHYv84uUlLCE5uEhPp+4CRI4E//gBu3gQOHQI6daK+Dd27yw6RgZMbxmxK1arUWoNPTVkeJzdp/fMP8OQJUKuW7EjkmTkTaNgQELf/pUSmQgVqRJU/P3DkCFCggOwQGXhwJmM2R62meVMvXqT0Y2Hmx8lNWrrOxI66cpOUBCxYADRrBqiW8zsMJeOVG8ZsTGAgbe9v3So7EvvGyU1aGg2VlPj5yY5Ejm3baPVq4EDZkbCscHLDmI0pV47eOfPWlGVxcpOWo9fbhIYCtWvTMFumbJzcMGaD1GpgyxYgNlZ2JPaLkxt9QlCPG0dNbq5do359vGpjGzi5YcwGdesGvHoFbN4sOxL7xcmNvrt3gfv3HTe5WbiQWtkEBsqOhBmCkxvGbFCJEjRvihv6WQ4nN/ocuZj45Utg6VKgd28u4rcVnNwwZqPUamDHDuDxY9mR2CcPDyAuTnYUyqHR0GnnYsVkR2J9a9bQv7P+/WVHwgzFyQ1jNqprV2ootmGD7EjsE6/c6NNoqL+NSiU7EusLDQVatqRifmYbOLlhzEYVKgS88w6fmrIUXXIjhOxIlMFRT0pFRgJ//82FxLaGkxvGbJhaTVPC79+XHYn98fAAtFogIUF2JPLdvw/8+69jJjdhYbQV17at7EiYMTi5YcyGde5M2wRr18qOxP7oCkd5awo4cYI+O1py8+QJsGoV8NlnQA7u529TOLlhzIb5+AAtWvDWlCVwcpMiMhLw9gZKl5YdiXWtXEl1bX36yI6EGYuTG8ZsnFoNHDhA2wbMfDi5SaHRADVrOlYxsVZLhcRdujjuuAlbxskNYzauY0fA1RVYvVp2JPaFk5sUjlhMvHcvcOUKFxLbKk5uGLNx3t5Amzbc0M/cOLkhT54AN244XnITGgpUrQo0aiQ7EmYKTm4YswNqNXD0KHD9uuxI7AcnN0RXTFyrltw4rOnff4FNm2jVxpG24uwJJzeM2YH27enFmFdvzIeTG6LRAJ6ejtXAbtEi+vl/9JHsSJipOLlhzA54elKCw6emzIeTG6LRAP7+gLOz7EisIyEBWLwY6NkTyJ1bdjTMVJzcMGYn1Grg9GngwgXZkdiHnDnpMyc3jlVvs3EjEBUFDBggOxKWHZzcMGYnWrcGvLx4a8pcXFzow5GTm2fPgMuXHSu5CQ0FmjShYmJmuzi5YcxOuLvTsfDwcJ6HZC6OPjzz5En6XXKU5ObcOeCPP/j4tz3g5IYxO6JWA5cuAadOyY7EPjh6cqPRAG5uQKVKsiOxjrAwwNcX6NRJdiQsu6QnN/Pnz0fJkiXh7u6OunXr4ujRo5lef+7cuahQoQJy5syJYsWKYfjw4Xj16pWVomVM2QICgHz5eGvKXDi5AapXp+05e/fsGfC//wF9+1JTTGbbpCY3ERERCA4OxqRJk6DRaFCjRg20atUK9zMYcbxq1SqMGTMGkyZNwoULF7B06VJERERg3LhxVo6cMWVycaF28bw1ZR6c3DhOf5tffgHi4oB+/WRHwsxBanIze/Zs9O3bF71790blypWxYMECeHh4YNmyZele/9ChQ2jYsCF69OiBkiVLomXLlujevXuWqz2MORK1Grh5k5r6sezx9HTc5ObFC+D8eceotxGCConffx8oVkx2NMwcpCU3CQkJiIyMREBAQEowTk4ICAjA4cOH071NgwYNEBkZmZzMXL9+HVu3bsV7771nlZgZswVNm1LdAPe8yT5HXrk5c4aGRzpCcvPXX/R8uZDYfuSQ9cAPHz5EUlISfH199S739fXFxYsX071Njx498PDhQzRq1AhCCCQmJqJ///6ZbkvFx8cjPj4++f9jY2PN8wQYUyhnZ6BbNxqkOWsW4CS9ss52OXJyo9EAOXI4xpHo0FDqwNy8uexImLnY1J+9/fv3Y+rUqQgNDYVGo8H69euxZcsWTJkyJcPbhISEwNvbO/mjGK85MgegVgN37wIHD8qOxLY5cnITGUmJjZub7EgsKzoaWLuWmvbxGwH7Ie1H6ePjA2dnZ0RHR+tdHh0dDT8/v3RvM2HCBHz88cf49NNPUa1aNXTq1AlTp05FSEgItFpturcZO3YsYmJikj9u375t9ufCmNLUq0e1A7w1lT2OnNw4SmfipUtphapXL9mRWNidOzQsK39+ar9drRpw/LjsqCxGWnLj6uqKWrVqYc+ePcmXabVa7NmzB/Xr10/3Ni9evIDTG6m1838DT0QGR0Pc3Nzg5eWl98GYvXNyAgIDgTVrgMRE2dHYLkdNbuLjgbNn7T+5SUoCFiwAuncH8uaVHY0FPXkCNGxIxym3baNK8Vmz7PpJS6u5AYDg4GAEBQWhdu3aqFOnDubOnYu4uDj07t0bANCzZ08UKVIEISEhAID27dtj9uzZqFmzJurWrYurV69iwoQJaN++fXKSwxgjajUwcyawdy/QsqXsaGyToyY3584Br1/bf3KzZQtw+7YDFBJPn05LucuXp1xWqpS8eKxAanITGBiIBw8eYOLEiYiKioK/vz+2b9+eXGR869YtvZWa8ePHQ6VSYfz48bhz5w4KFCiA9u3b49tvv5X1FBhTrLfeAsqWpYZ+nNyYxlGTG42GVv9q1JAdiWWFhgJ16jhAL5/Nm4FWrYCuXWm+RJEilNH17Ss7MotRiYz2c+xUbGwsvL29ERMTw1tUzO6NHw/Mn09Tju29MNQSpk8HZswAHj2SHYl1DRwI/PknbU3Zq6tX6YTUihVAUJDsaCzM3Z0+BwdTgnPsGPD557QnZ0NP3pjXb64NZ8yOqdXA06fAzp2yI7FNjrxyY+9bUgsW0KiSbt1kR2IFuoZFU6cCNWtSG+a+fembYKc4uWHMjlWtClSpwqemTOXhAbx6Ra8NjiIxkQav2nNy8/IlsGwZ8MkndHDI7hUqBFSurH9ZpUrArVty4rECTm4Ys3NqNbBpk2OuQGSXhwd9fvlSbhzWdOECJXT2nNxERNABov79ZUdiJQ0bApcu6V92+TJQooSceKyAkxvG7FxgIA0E3LpVdiS2R5fcOFJiqNHQZ39/qWFYVGgo0Lo1UKaM7EgsKyKCmnli+HDgyBHalrp6FVi1Cli0CBg0SHaIFsPJDWN2rlw5ehfOW1PGc9Tkplw5wF7PWxw7Rh/2fvz7zBnq2bdsGYC33wY2bAB+/ZX2qqdMAebOBT78UHaYFiP1KDhjzDrUamDiRCA21n5ftCzBUZMbe96SCgsDihcH7HneclIS1QuXKweMGvXfhe3a0YeD4JUbxhxAt25UR7F5s+xIbIujJTdaLXDihP32fXn8mBYv+venAbP2av584OhRYMkSx20BwckNYw6gRAmgQQPag2eGc7Tk5soVqs+y15WbFStoVaNPH9mRWM6tW8C4cbTt1qCB7Gjk4eSGMQcRGAjs2EHvXplhHC250RUT16wpNw5L0GppS6prV6BgQdnRWIYQNN08Tx6qHXZknNww5iC6dqUeJhs2yI7EdjhiclOyJDW3sze7d9NBIXsuJI6IoFORoaFcW8fJDWMOolAh4J13+NSUMRwtuYmMtN8tqdBQoHp1+92qefQIGDqU3sS8/77saOTj5IYxB6JW05Tw6GjZkdgG3UgeR0huhLDfk1K3bgG//UarNiqV7GgsY+RImuT+ww+yI1EGTm4YcyCdO9O053XrZEdiG1Qqx5kvdeMGEBNjn8nNokWAp6f9tnXZvZuKpWfNAvz8ZEejDJzcMOZAfHyAgADemjKGoyQ3umJie0tuEhKAxYtp+HWuXLKjMb8XL4DPPgOaNQN695YdjXJwcsOYg1GrgQMHgH//lR2JbXCk5KZIEcDXV3Yk5rV+PXD/Pp0iskdffUUjFhYutN8tN1NwcsOYg+nYEXB1BVavlh2JbXCk5MbeVm0AKiR+5520Q7HtgUZDW1GTJlE3YpaCkxvGHIy3N7We54Z+hnGE5MZei4nPnKFVSns8/p2YCHz6KVCtGjBihOxolIeTG8YcUGAgtWe/fl12JMrnCMnNnTvAgwf2l9yEhVGBbceOsiMxvzlzgFOnaMSCi4vsaJSHkxvGHFD79vSizas3WXOE5CYykj7bU3ITGwv89BPQr5/9vfhfu0aDcIcNA2rXlh2NMnFyw5gD8vSkBIdPTWXNEZIbjQYoUIAKiu3Fzz8DL1/SdGx7IgSdjvLzA77+WnY0ysXJDWMOSq0GTp8GLlyQHYmyeXjQMEl7pqu3sZfTNkJQIXGHDkDRorKjMa+VK4E9e+h0lKen7GiUi5MbxhxU69Y0f4a3pjLnKCs39rQldeAAcO6c/RUSR0cDwcHAxx8DLVvKjkbZOLlhzEG5u1OhZXg4vdNl6bP35CYqivqk1KolOxLzCQ0FKlQA3n1XdiTmNWwY4OwMzJ4tOxLl4+SGMQemVgOXLtGpC5Y+e09uTpygz/aychMVReNFBgywn202ANiyhd6IzJ1LncZZ5ji5YcyBBQQA+fJxYXFm7D250WiAPHmAkiVlR2IeuqPRQUGyIzGfZ88oWWvVCujRQ3Y0toGTG8YcmIsL8MEHVHfDW1Ppc4Tkxl6KiRMTqdD2ww8pYbMXX34JPHoELFhgHz8na+DkhjEHFxgI3LxJTf1YWrrkxl6Tv8hI+9mS+v13mplmT3OkjhwB5s0Dvv3WflbXrIGTG8YcXNOmNCyRt6bS5+EBJCUBr1/LjsT8Hj0C/vnHfpKb0FCgXj37eT4JCTRioXZtYMgQ2dHYFk5uGHNwzs5At240SFOrlR2N8nh40Gd73Jqyp2Liy5eBXbvs6/j3jBlU8L94Mf07ZYbj5IYxBrWajgMfPCg7EuWx5+RGowFy5bKPidILFgD58wNdu8qOxDwuXgSmTAFGjQJq1JAdje3h5IYxhnr1gOLFeWsqPfae3NSsCTjZ+CvBixfA8uVAnz7Uv8nWabU0NqJECWDCBNnR2CYb/5VmjJmDkxNtTa1ZQydOWAp7T27sYUsqPByIiaGZS/Zg8WJaRV28GMiZU3Y0tomTG8YYANqaevgQ2LtXdiTKYq/JTWwscOWK7Sc3QgDz5wNt2gClS8uOJvvu3AFGj6ZC4qZNZUdjuzi5YYwBoBe5smV5a+pNuuGE9pbcnDxJn209uTl2jFag7KWQeMgQSqhnzJAdiRmULEmNed78GDTI4g/NyQ1jDAD9zVGrgQ0bgPh42dEoh72u3ERGUn1KxYqyI8me0FB6DW3dWnYk2bd+Pf37+/FHIG9e2dGYwbFjwL17KR+7dtHlVqj65uSGMZYsMBB4+hTYuVN2JMphr8mNRkOncHLkkB2J6R49opXG/v1t/6j006fA4MFAhw5Aly6yozGTAgUAP7+Uj99/B8qUscp+Gyc3jLFkVasCVarw1lRquoJOe0xubH1Lavlyqrn55BPZkWTfF18Az59T/ZBdjlhISAB+/pl+WFZ4gpzcMNP9+SfQvj1QuDD9sm7cKDsiZgZqNbBpk/29mJvKxYU+7On7ERdHfVRsObnRaoGwMDrlV6CA7Giy548/gEWLgOnTgSJFZEdjIRs30vJUr15WeThObpjp4uJoXXv+fNmRMDMKDKQf7datsiNRDnsbnnn6NCUHtWrJjsR0O3cC16/bfiHxq1dAv35Aw4b2c5Q9XUuX0pG2woWt8nA2vNvKpGvThj6YXSlXjl70wsNpYjizv+RGo6HVqCpVZEdiutBQwN+fGlDasm++ocG1GzfafjPFDP3zD7B7N1VMW4m9fisZY9kQGAhs2UK9UJh9JjfVqgGurrIjMc3Nm1SbOnCgbdennDlDW1FffglUqiQ7GgtavhwoWBBo29ZqD8nJDWMsjW7daLl882bZkSiDPSY3tlxvs2gRkDs30KOH7EhMl5REjfrKlwfGjJEdjXmdO0fPDwDtfy5fDgQFWfVoHic3jLE0SpQAGjQAIiJkR6IM9pTcvHoFnD1ru8lNfDywZAnVpeoaLNqiefOoDcySJba7gpYejQaoWxeYNeu/C3bvBm7dsvqRNk5uGGPpUquBHTuAx49lRyKfPSU3Z8/S/DBbTW7WrQMePAAGDJAdien++Ye2ogYOBOrXlx2N+dy4Abz3HtVyJTchbtmSzuuXL2/VWDi5YYyl64MP6EVwwwbZkchnT8mNRkMN76pXlx2JaUJDgXfftd3OykJQYpY3LzB1quxozOfRIzpfkisX8Ntv8lfVOLlhRnv+PNV/nDyZMqTmxg3671u35ATGzKpQIeCdd7ihH2B/yU2lSrY5bfrUKeCvv2z7+Hd4OLBtG/Xo8fKSHY15vHwJvP8+JTjbt1PtsGyc3DCjXLtG3bM3bABw/DhQsyZ9AEBwMP33xIlSY2Tmo1bTlPDoaNmRyGVvyY2t9rcJC6M2Ke+/LzsS0zx8CAwdSgX77drJjsY8kpKADz8ETpygE2xly8qOiHByw4xSqhTQrBnQvTuwH+/QGuubHytWyA6TmUnnztR7Y9062ZHIZS/JzevX1MDPFuttYmKoe3+/ftSjxxaNGEHJwA8/yI7EPIQAhg2jjuarV1MhsVJwcsOM4uQE/O9/QJMm9O7pxAnZETFL8vEBWrTgrSl7SW4uXKDTRraY3Pz0E5306ttXdiSm2bWL/nbOnAn4+sqOxjy++45OfYWFKW8lipMbZjRXV2o0WbEi0Lo1cOWK7IiYJQUGAgcOAP/+KzsSeewludFoqOldjRqyIzGOEFRI3KmT1br3m1VcHI1WePddoHdv2dGYx6pVNOxz/HhaTVMaTm6YSXLlotlDefPSSb+7d2VHxCylY0dKaFevlh2JPPaS3ERG0onc3LllR2KcP/6gVSdbLST+6ivg3j1g4ULb7qiss3cv9RkKCgK+/lp2NOnj5IaZzMeHhtclJtIKztOnsiNiluDtTb0rHLmhn70kN7bamTg0lFaK33lHdiTGi4wEZs+mBEcpxbbZcfo0raA1awYsXqzcZI2TG5YtxYtTo7c7d4D27e3jBYClpVYDR4/SFGZH5OFBx121WtmRmC4piTo12Fpyc/cunc60xTlSr1/TiIXq1ekwqa27fZve6JQpA6xdq+zCbk5uWLZVrkxDFjUaqs94/Vp2RMzc2rWjF3hHXb3x8KDPr17JjSM7Ll+mNx+2ltzoxhP07Ck7EuPNmUMrHUuWKDsRMMTTp9SkL0cO+nuv9K1NTm6YWdSrR8eFt2+n0wy2/A6XpeXpSStzjnpqSpfc2PLKpEZDn20puXn9mupUPvqItkdtydWrwKRJwPDhtttXSCc+nmrv7t6lBoSFCsmOKGuc3DCzad2ajjquXElV9My+qNX0LvTCBdmRWJ+9JDelSwN58siOxHC//UYvqLY2R0oIOh3l5wdMniw7muzRaqlw+MgR+nlUqiQ7IsNwcsPMqnt3alA1cyYwY4bsaJg5tW5N7eIdcWvKXpIbW1q1AYD582k6vb+/7EiMs2IFnShauFD+jKXsGj2aTkquWgU0bCg7GsNJT27mz5+PkiVLwt3dHXXr1sXRo0czvf7Tp08xaNAgFCpUCG5ubihfvjy2bt1qpWiZIYYMASZMoNWbZctkR8PMxd2dTkmEh9M7U0di68mNVmt7yc2FC5Qg2Nrx7+ho6kTcsye1ybBl338PzJpFnzt3lh2NcaQmNxEREQgODsakSZOg0WhQo0YNtGrVCvfv30/3+gkJCWjRogVu3ryJtWvX4tKlS1i8eDGKFCli5chZViZPpmXZvn2BzZtlR8PMJTAQuHSJBhg6EltPbq5fB2JjbSu5WbCA2k188IHsSIzz+ec0dX3WLNmRZM+6dVQvNHIkvWG1OUKiOnXqiEGDBiX/f1JSkihcuLAICQlJ9/phYWGidOnSIiEhweTHjImJEQBETEyMyffBDJOYKMQHHwjh5ibEH3/IjoaZQ0KCEPnyCfHFF7Ijsa5//qHBaTt2yI7ENBERFH90tOxIDPP8uRBeXkKMGSM7EuP89ht9n3/5RXYk2fPnn/R3W60WIilJdjQpjHn9lrZyk5CQgMjISAQEBCRf5uTkhICAABw+fDjd22zevBn169fHoEGD4Ovri6pVq2Lq1KlISkrK8HHi4+MRGxur98Gsw9mZBt01bEgnbRzt3b49cnGhd9IREY61NWXrKzcaDVC0KFCwoOxIDLNqFfDsGa3+2opnz6jwuXVrqj20VRcuAB06UK3TihU0T9AWSQv74cOHSEpKgu8bE8R8fX0RFRWV7m2uX7+OtWvXIikpCVu3bsWECRMwa9YsfPPNNxk+TkhICLy9vZM/ihUrZtbnwTLn5gZs3AiUKwe0agVcuyY7IpZdajVw8yY19XMU9pDc2MqWlG6OVNu2QMmSsqMx3LhxwJMntJ1ma80Gde7epeSsSBGaH+jmJjsi09lUTqbValGwYEEsWrQItWrVQmBgIL788kssWLAgw9uMHTsWMTExyR+3b9+2YsQMoGZP27ZRn4qWLYEMcldmI5o0oSOujtTzxt2dPtticiMEJTe20mvlyBHqpGxLhcSHD9PJrm++AUqUkB2NaWJjKaFMSqK5gbbUMiA9JiU3QUFB+PPPP7P1wD4+PnB2dkZ0dLTe5dHR0fDz80v3NoUKFUL58uXh7OycfFmlSpUQFRWFhISEdG/j5uYGLy8vvQ9mfQUK0ByqV694DpWtc3YGunal46GO0qzRyQnImdM2k5vbt4FHj2xn5SY0FChVilZ6bUFCAh2cqF3bRgtvQc/hgw+o8HzbNsAeNjhMSm5iYmIQEBCAcuXKYerUqbhz547R9+Hq6opatWphz549yZdptVrs2bMH9evXT/c2DRs2xNWrV6FN9Rf18uXLKFSoEFxdXY1/IsyqSpSgBOfWLdrTfflSdkTMVGo1LWEfPCg7Euux1eGZttSZ+MEDSpoHDLCdWo/p0+kE4ZIllPjbGiEoOfvjDyohqFZNdkTmYdKvz8aNG3Hnzh0MGDAAERERKFmyJNq0aYO1a9fitRGDhYKDg7F48WKsXLkSFy5cwIABAxAXF4fevXsDAHr27ImxY8cmX3/AgAF4/PgxPv/8c1y+fBlbtmzB1KlTMWjQIFOeBpOgShXg99+BY8eo6C4xUXZEzBT16tHQVEfamrLl5MbX1zZa5i9bRvUq/70EKN6FC7QVNXo0Dce0RRMmUGf5FSto0rfdMMfxrMjISDF48GDh7u4ufHx8xLBhw8Tly5cNuu2PP/4oihcvLlxdXUWdOnXEkSNHkr/WtGlTERQUpHf9Q4cOibp16wo3NzdRunRp8e2334rExESDY+Wj4MqwdasQOXII8cknQmi1sqNhphg1SggfHyFev5YdiXVUqCDEiBGyozDee+8J0aaN7CiylpgoRMmSQvTsKTsSwyQlCdGwoRDlywvx8qXsaEyzYAEdXZ8xQ3YkhjHm9Tvbyc3du3fFtGnTRIUKFYSnp6fo2bOnaN68uciRI4eYPXt2du/e7Di5UY6ffqJ/WI7WM8VeHD9u271fjFWzphADBsiOwnh+fkJ8+aXsKLL2++/0+5Tq/a2ihYVRvPv3y47ENJs2CeHkJMSQIbbzBtPifW5ev36NdevWoV27dihRogTWrFmDYcOG4e7du1i5ciV2796N1atX4+uvvzbnIhOzMx99BMyZQ3vWtt7N0xG99RZQtqzjbE15eABxcbKjMM69e3Q60RbqbUJDKc46dWRHkrU7d2grqm9foGlT2dEY78gRqpvr2JH+Btvq0fXM5DDlRoUKFYJWq0X37t1x9OhR+Kcz1axZs2bIY+tnyZjFDRtGRYQjR1Kr9aAg2RExQ6lU9Ady3jwgLMy2e2IYwhZrbmylmFh3SmfxYuW/0AoBDBpEAzFtcTjwlSvUVPWtt6jJqi0WQRvCpORmzpw56Nq1K9x1zR/SkSdPHty4ccPkwJjj+OYbSnD69AHy5aN/eMw2qNX089u50/5/braa3OTLp/zeKwsX0sR5W+jsu349sGkTsHat7fWCuX8faNMGyJ+fnkPOnLIjshyjt6Vev36N3r174+rVq5aIhzkglYre+XfoAHTr5ljHi21dlSr04QhbU7aa3Lz1lrJXQ169ApYupRNSuk7QSvXkCTB4MG3n2NqU7Lg4oF07+rx9OyU49szo5MbFxQXFixfPdJ4TY8ZydgZ++QWoX5/+AZ4+LTsiZii1mt4F2toLv7FsOblRsjVrqMlg//6yI8naF1/Q78C8ecpOGN+UmEhvHC9coO7DtjTWwlQmFRR/+eWXGDduHB4/fmzueJgDc3enJlKlS1MXY97VtA2BgfRucOtW2ZFYlq0lNw8fUsNMpSc3oaFAQABQoYLsSDL3xx9UEzR9Os1eshVC0CiLnTuBdeuAmjVlR2QdJtXczJs3D1evXkXhwoVRokQJeHp66n1do6tiY8xIXl5UWNioEc2hOniQGpAx5SpXjuYWhYdTC3d75elpW8mNLRQTazR0cmf9etmRZO7VKzoZ1agR0K+f7GiM8803lJQtX05/Ux2FSclNx44dzRwGYyl8feldRsOGVPy2bx8N3WTKpVZTp9PYWEpQ7ZGtrdxoNDS0tkwZ2ZFkLCyMVkGUXow+ZQrwzz+0/WorYyEASmgmTqT4e/WSHY11mZTcTJo0ydxxMKanVClgxw6aQN2xI63mZHI4j0nWrRswahSweTP1L7JHtpjc1Kyp3Bfjp0+pzm7sWCCHSa9E1nH6NB35njgRqFRJdjSG276dVpv69QO+/FJ2NNan0F97xmiA22+/0bJ1jx48h0rJihcHGjSw71NTtpjcKHlLauVK4PVr4NNPZUeSsaQkiq9CBSomthUaDW0Rt2kDzJ9vW8XP5mJScpOUlISZM2eiTp068PPzQ758+fQ+GDOXRo3oNMXmzTQpWAjZEbGMqNW0nWiv5ww8PCjBNmI2sDRPnwLXrlEtlBIJQYXEnTsre6Dnjz8Cx49TzYqrq+xoDHPjBvDeeyktGpS8KmZJJiU3kydPxuzZsxEYGIiYmBgEBwejc+fOcHJywldffWXmEJmja9eOpgUvWQKMHy87GpaRrl3pne6GDbIjsQxdDxZbWL05eZI+K3XlZu9e4PJlOsWjVDdv0t+bQYOoRYUtePSIVmty5aJV7zfO+jgUk5KbX375BYsXL8aIESOQI0cOdO/eHUuWLMHEiRNx5MgRc8fIGHr2BGbOBKZOBebOlR0NS4+fH83ZsdetKVtKbjQa6j6r1OPVoaFA5cpUU6dEQtBKcd689DfHFrx8SYXZjx9TvU3BgrIjksuk5CYqKgrVqlUDAOTKlQsxMTEAgHbt2mHLli3mi46xVEaMoH3v4cNpJgpTHrWa3pVHR8uOxPxsLbnx91fm3KB//6VTRwMHKrcW5NdfKUEIC6MTZ0qXlAR8+CGt2P3+Ow20dXQmJTdFixbFvXv3AABlypTBzp07AQDHjh2Dm71Pz2NShYQAn3xCrdrtvWmcLercmU7nrFsnOxLzs6XkJjJSuVtSixfTycePP5YdSfoePgQ+/5yaU7ZrJzuarAlBA4g3bQJWr7aNqerWYFJy06lTJ+zZswcAMGTIEEyYMAHlypVDz5498cknn5g1QMZSU6loyF7btnQa4NAh2RGx1Hx8gBYt7HNrylaSm+fPgUuXlJncvH4NLFpEiY1S+yEFB9NKyPffy47EMN99R+MgwsJsIxmzFpPqqKdNm5b834GBgShevDgOHz6McuXKob3SuzExm5cjBy0bt25NSc6BA0DVqrKjYjqBgdQw7N9/gaJFZUdjPraS3Jw6Re/mlZjcbNwIREVRPYsS7dwJ/PQTHWCwhc7oq1bRVv348bbXOdnSVEI41uHa2NhYeHt7IyYmBl5KfevADBITQwWsDx4Af/3lGMPgbEFMDBUzhoTQu2B7ce8eULgw1TS0bSs7moz9+CMwciTw7Jnyji83a0bH6Q8ckB1JWnFx9CapdGlg927l1gPp7N1Lb/B69KBOxEqP1xyMef02+QT8lStXsG/fPty/fx9arVbvaxMnTjT1bhkzmLc3Ff01bJgyh8rRTwgogbc39dmIiLCv5MZWVm40GqB6deUlNufPA/v302qDEk2aRKtKu3YpP1E4fRro1ImSxcWLlR+vDCYlN4sXL8aAAQPg4+MDPz8/qFJ9Z1UqFSc3zGr8/FLmUL33Hs2hsoXTDfZOraaP69fpnbA9sKXkpl492VGkFRZGbz46d5YdSVrHjwNz5tBqo9JPGt2+TX/rypQB1q4FXFxkR6RMJhUUf/PNN/j2228RFRWFkydP4sSJE8kfPBGcWVuZMjSH6soVmkMVHy87ItauHSUDERGyIzEfFxeq91JycvPqFXDunPLqbZ4/p3ELn34KKO1A7evXNIOpenXlrzQ+fUpN+nLkALZs4TdymTEpuXny5Am6du1q7lgYM1mNGtSR89Ah6veQlCQ7Isfm6UkNxezt1JTS50udOUO/+0pLbn75hWpalFj0Ons2bfMsWaLsUQXx8fTm7e5d2o5X8tgKJTApuenatWtybxvGlKJJE1op2LiRWqY7Vqm88qjV9KJx4YLsSMxH6clNZCQ17vuvx6oi6OZItWsHlCghOxp9V68CX31FKzZKncMFAFotEBREQ4R/+w2oWFF2RMpnUp5atmxZTJgwAUeOHEG1atXg8sam39ChQ80SHGPGev99KrD75BOgQAFgyhTZETmu1q2pl0lEBL2A2AOlJzcaDQ1MdHeXHUmKQ4coyZ0xQ3Yk+oSglaRChYDJk2VHk7nRo6lB39q1VF/IsmZScrNo0SLkypULf/zxB/744w+9r6lUKk5umFS9e1OX0dGjKcHhX0c53N3pREd4OJ1EsYcTHbaQ3ChtSyo0lOriWrSQHYm+5cvpAMLOnSnF4kr0/ffArFnADz8osxhbqUxKbm7cuGHuOBgzq1GjgPv3qY26jw/1gmDWFxhIhaSnTtGsI1un5OQmIYFqboKCZEeS4v59YM0aOoXkZFIRhGVERVEvoJ49lZd0pbZ2Lc3SGzUKGDJEdjS2RcHlU4xlz4wZ1OAvKAjIl4+2SZh1BQTQ9z48nJMbSzt/nhIcJdWOLF1KNUC9esmORN/nn1Px8OzZsiPJ2IEDwEcf0RuEVEMBmIEMTm6Cg4MxZcoUeHp6IjiL83KzlfwbwxyGSkUnIB49Arp0AfbsUWb/D3vm4kIzwCIi6N27rW9NKTm50Wjo+1ujhuxISFISsGABFZbnzy87mhS//Ub1K6tWKSuu1C5cADp0ABo0AFasUNaql60wOLk5ceIEXr9+nfzfGVHZ+l8vZldy5KA/ZC1bpsyhqlxZdlSORa2mYYlHjwJ168qOJns8PIDYWNlRpE+joVM0np6yIyFbtwK3bgEDB8qOJEVsLMXTpg39XirR3bu0ylykCLB+vfL6AtkKg5Obffv2pfvfjCldzpz0bq1JE0pyDh0CiheXHZXjaNKEOkmHh9tHchMVJTuK9CmtmDg0FKhdG3j7bdmRpBg3DnjyhLolK/F9eGwsdR9OSgK2bQPy5JEdke3ixS7mEPLkoS7Grq6U4Dx4IDsix+HsDHTtSltTtt5cUanbUomJwMmTyklurl2jRnNKWrU5dIgSrm+/VV6/HYDqpbp0AW7coMSmaFHZEdk2kwqKO3XqlO72k0qlgru7O8qWLYsePXqgQoUK2Q6QMXMpVEh/DtXevdy+3FrUappWffAgTXK3VUpNbi5dAl6+VE5ys2ABkDcvFcMqQXw8jVh4+21g8GDZ0aQlBMX355+UFCqpCaOtMmnlxtvbG3v37oVGo4FKpYJKpcKJEyewd+9eJCYmIiIiAjVq1MBff/1l7ngZy5ayZemPx6VL1DOC51BZR716tBVo67OmlJrc6Eb6KeFE2suXwLJl1G9KKf1jpk8HLl+mAwbOzrKjSWv8eOB//6O2Cc2ayY7GPpiU3Pj5+aFHjx64fv061q1bh3Xr1uHatWv46KOPUKZMGVy4cAFBQUH44osvzB0vY9lWsyaweTMVF/fsaftbJbbAyYnexa9ZQ1sotkrJyU2ZMsqo0Vi9Gnj8GOjfX3Yk5Px54JtvgC++UOaKyIIFwNSpwHffKbfI2RaZlNwsXboUw4YNg1Oq82lOTk4YMmQIFi1aBJVKhcGDB+Ps2bNmC5Qxc3rnHeDXX6lJ1tChPIfKGgIDqXP03r2yIzGdkpMbpfS3CQ2lurZy5WRHQjOZ+vYFSpWi1RGl2byZ5uANGQKMGCE7GvtiUnKTmJiIixcvprn84sWLSPrvbbC7uzsfC2eK1qkTsHAh/TFW+mwZe/DWW7QtaMuTwnXJjZKSYa0WOHFCGfU2x4/TkX+lFBIvXEiFxIsXK2veFkBDMNVqmvQ9Z44yT2/ZMpMKij/++GP06dMH48aNw9v/nfM7duwYpk6dip49ewIA/vjjD1SpUsV8kTJmAZ9+SqsJY8fSHKpBg2RHZL9UKvpjPm8eHcW1xf4duhqSV6+oxYASXLsGPHumjOQmLAwoVox6Ssn277+0FdWvH7UjUJIrV4D27eln9vPPyqwDsnUmJTdz5syBr68vZsyYgejoaACAr68vhg8fnlxn07JlS7TmfvfMBnzxBc3AGTKEOpbyvrflqNVU/7BzJ/1xtzW65ObFC+UkN7pi4po15cbx5Al1/R0/nppnyiQEvVHJlYuKiZUkOpqa9Pn40LaUUn6P7I1Jv4LOzs748ssv8eWXXyL2v3adXl5eetcpzl3SmI1QqYCZM6n3Tc+eNAupZUvZUdmnKlWAqlVpa8rWkxultO6PjKSTaD4+cuNYsYKK8/v0kRsHAKxbR4nDunXKKLLWiYsD2rWj3589e+hvDbOMbDfx8/LySpPYMGZrnJzo+GqLFnRE/OhR2RHZr8BAYNMmZRbmZiV1cqMUSuhMrNXSllSXLtSNWqYnT6iXTceO9G9ZKRITgW7dgIsXaTRFyZKyI7JvJic3a9euRbdu3VCvXj289dZbeh+M2SIXFzqqXKMGNfm7cEF2RPYpMJDewW7dKjsS4yktuRFCGcnNnj1UR6KEQuLRo6nXzrx5siNJIQQwYABtx65bJ38L0RGYlNz88MMP6N27N3x9fXHixAnUqVMH+fPnx/Xr19GmTRtzx8iY1Xh4AL//Tt2MW7UCbt+WHZH9KVeOji3b4qkppSU3//xDKxWyk5vQUNpubNRIbhz791OjvhkzaPCkUkyZQnEtWcJb3tZiUnITGhqKRYsW4ccff4SrqytGjx6NXbt2YejQoYiJiTF3jIxZVd681MXYyYkSnEePZEdkf9RqYMsW5U7YzojSkhtdMbHMHje3b1N9y8CBco8zv3xJJ6MaN6beNkqxfDkwaRIV0gcFyY7GcZiU3Ny6dQsNGjQAAOTMmRPPnj0DQEfEf/31V/NFx5gkRYrQEvLDh3Ss9flz2RHZl27d6Dj15s2yIzGOEpObQoXk1rksWkTfl48+khcDQKsj//xD8Thlu5rUPLZvp0SrXz+aSM6sx+TxC48fPwZAp6KOHDkCALhx4waEkrpbMZYN5cvTdN5z56hQMiFBdkT2o3hxoEED29uaUmJyI3NLKiGBGuT17Cl3CO2pU7QVNX48ULGivDhSi4wEPviA6vfmz+cmfdZmUnLz7rvvYvN/b7l69+6N4cOHo0WLFggMDESnTp3MGiBjMtWqRSd79u+nJWWtVnZE9kOtptWx/94n2QRdTxIlJDdC0AuozORmwwbq2zJggLwYkpKoGWfFitSzSglu3KAV3ypVaMyL7L4/jsikb/miRYug/e+v/KBBg5A/f34cOnQI77//Pj777DOzBsiYbO++S83JunalXiI//MDvwsyha1dg2DB6gVRCbxRDODlRG38lJDd371LzSZnJTWgodf+tWlVeDD/8QEneoUOAq6u8OHQePaImfblz0+EET0/ZETkmk5IbJycnvaGZarUaam7ryuxYly40vfezz4CCBYEJE2RHZPv8/GiAaXi47SQ3gHKGZ+qKiWUlN2fPAn/+KXdr8eZN2ooaPBioV09eHDovX1JzyidPKNkqUEB2RI7L5MWyV69e4fTp07h//37yKo7O+++/n+3AGFOafv2oi/H48fRHq39/2RHZvsBA2tKIjgZ8fWVHYxhPT+rTI5tGQ12SixWT8/hhYfQzk1WJIAT9G8yfH/j2WzkxpJaUBPToQfU/+/bRkFgmj0nJzfbt29GzZ088fPgwzddUKlXyZHDG7M24cbQVMHAg/VHt2lV2RLatc2eaAbRunTIawBlCSSs3b70lZ4v02TPgf/+jbUVZW0GrVgE7dtDWj8xiZoASrc8/p9N/mzYBderIjYeZWFA8ZMgQdO3aFffu3YNWq9X74MSG2TOVCpgzB+jeHfjwQ2D3btkR2TYfHxp5YUunppSW3Mjw88/0PejXT87jP3hAyYRarYwJ5N99RyeiwsJodhSTz6TkJjo6GsHBwfC1lXVkxszIyYkaczVvTkvyx4/Ljsi2qdXAgQPAv//KjsQwSkhu7t+n75eM5n1CUCHx++/L2xILDqaTi3Pnynn81FatolNa48fLS/ZYWiYlNx988AH2799v5lAYsx2ursDatXRKpE0b4NIl2RHZrg4d6Pu5erXsSAyjhOTmxAn6LGPl5uBBKiaWtY24YwetHM2eLb9Oa88eoFcv+vj6a7mxMH0qYULXvRcvXqBr164oUKAAqlWrBhcXF72vDx061GwBmltsbCy8vb0RExPD08xZtj1+TO3enz+n0xFKmmdjSzp1Au7csY1p7B070oTn33+XF0NICDB9Op3KsXbNTffudPT64kXrdwKOi6M3FGXKALt2yW3JcPo0/duvXx/47TcavMssy5jXb5MKin/99Vfs3LkT7u7u2L9/P1SpfsNUKpWikxvGzClfPnon2bAhDcQ7cIAuY8ZRq+nj+nWgdGnZ0WTOwwOIipIbQ2QkTZa29ot7VBQVf0+fLmfEwcSJdLJu9265ic3t27RiW6YMsGYNJzZKZNKv55dffonJkycjJiYGN2/exI0bN5I/rl+/bu4YGVO0okWp0+79+1RMqIRjwramXTtKGiIiZEeSNSVsS8kqJl66lLrt9upl/cc+doxqbCZPpqRClqdPKbFxcaHhr7JParH0mZTcJCQkIDAwUK+RX3bMnz8fJUuWhLu7O+rWrYujBq5Nh4eHQ6VSoWPHjmaJgzFTVagAbN1KS9VduwKvX8uOyLZ4elKBqi2cmpKd3Dx5Qu39rZ3cJCYCCxfStlTevNZ97NevacRCjRrA8OHWfezU4uNpW/LePRqKWaiQvFhY5kzKToKCghBhprdYERERCA4OxqRJk6DRaFCjRg20atUK9+/fz/R2N2/exMiRI9G4cWOzxMFYdr39NrBxIy2Z9+7Nc6iMFRhIyeGFC7IjyZzs5EZWMfGWLbQdI6OQeNYsGmC7ZIm8OU1aLQ0I/ftv6mejlAGdLH0m/ZokJSVhxowZ2LFjB6pXr56moHj27NkG39fs2bPRt29f9O7dGwCwYMECbNmyBcuWLcOYMWMyfPwPP/wQkydPxoEDB/D06VNTngZjZhcQQCc51Grq4TJnDs+hMlTr1oCXF21NffWV7GgyJju50Whopat8ees+bmgoNaez9vHzK1fo92H4cLlztEaPpvqatWupxo4pm0nJzZkzZ1CzZk0AwNmzZ01+8ISEBERGRmLs2LHJlzk5OSEgIACHDx/O8HZff/01ChYsiD59+uDAgQOZPkZ8fDzi4+OT/z82NtbkeBkzRLduNDxv4ECaQzVunOyIbIO7O52aCg8HJk1SblKohOTG3x9wdrbeY165QnVlK1ZY7zEB6qnTrx+dQpw82bqPndr339Pq0Q8/UFdtpnwmJTf79u0zy4M/fPgQSUlJaZoB+vr64uLFi+ne5uDBg1i6dClOnjxp0GOEhIRgssx/FcwhDRhAXVS//JLmUPXtKzsi26BWAytX0nwef3/Z0aRPCclNy5bWfcwFC+gUYLdu1n3cZcuA/fvp2LeHh3UfW2ftWlo1GjUKGDJETgzMeEYlN50NSFlVKhXWrVtnckCZefbsGT7++GMsXrwYPj4+Bt1m7NixCA4OTv7/2NhYFJPVVpM5lAkTKMHRDffjd3xZa96cvlfh4cpObl6/pg9rHwF+9gy4fBnIYMfeIl68oI7cffoAOXNa73GjooCRI4GgINruleHAAeCjjyjpnjZNTgzMNEYlN97e3mZ9cB8fHzg7OyM6Olrv8ujoaPj5+aW5/rVr13Dz5k20b98++TLdRPIcOXLg0qVLKPPGGUE3Nze4ubmZNW7GDKFS0XL2w4d0wmT7dqBZM9lRKZuLC9ClC9XdhIQoc2tKt4Lw8qX1k5uTJ2mrxpq1JxERdEKrf3/rPSYADB1K399Zs6z7uDrnz9MJvgYNKLmT0deHmc6o5Gb58uVmfXBXV1fUqlULe/bsST7OrdVqsWfPHgwePDjN9StWrIgzZ87oXTZ+/Hg8e/YM33//Pa/IMMVxcqJtlsePaczA/v1yiyJtgVoNLFpE3Yrr1pUdTVq65ObFCyqAtiaNBnBzAypVst5jhoZSsbc1e8ts3kzFu6tW0Uqetd29S71sihYF1q+n7zmzLZIO1aUIDg5GUFAQateujTp16mDu3LmIi4tLPj3Vs2dPFClSBCEhIXB3d0fVqlX1bp8nTx4ASHM5Y0rh6kpdXZs3pxeJv/4CypWTHZVyNWkC+PnR1pTSkxtr02iA6tWtt2J07BgNht282TqPBwCxsVSM/957lOhaW2wsPbZWC2zbBvz3EsNsjPSFtsDAQMycORMTJ06Ev78/Tp48ie3btycXGd+6dQv37t2THCVj2ZMrF/UJyZ+fikHv3pUdkXI5O1PhakQEkJQkO5q0ZCc31lz5Cw0FihenF3trGTuWugCHhVl/WzIhgbZFb96kxKZoUes+PjMfkwZn2jIenMlkunWLemTkyQP8+af1O73aikOH6Pu0fz/QtKnsaPSdPk2dcv/+m/q+WMuLF9TqPyyMjkdb2qNH9OI+cSIlHNbw1180jHLuXKq5sSYhqHg5IoLmxb3zjnUfn2XNmNdv6Ss3jDmS4sWpX8jdu0D79vJnFClVvXr0vVLirClZKzdnztBWibWa6K1YQStnffpY5/Hi46llQp06wKBB1nnM1MaPB376iWrkOLGxfZzcMGZllSrRHKqTJ2n7hedQpeXkROMY1qyhmUZKIiu50Who9IA1ygu1Wloh6tqVGlFaQ0gINQtcvNi6DQoB6uMzdSrw3Xdy6nyY+XFyw5gEdevSKYydO+mdMc+hSkutpmP0e/fKjkSfzOSmalXrnNzZtQu4ds16c6TOn6fkYswYoFo16zymzqZNtFI0dCgwYoR1H5tZDic3jEnSsiUtgf/8M3U/dazqt6zVrAmULau8SeGykpvISOsVE4eG0qmsBg0s/1haLW1HlS5NHb2t6cgR6kHVqRMwe7Yy+yox03Byw5hE3bvTvJrZs4EZM2RHoywqFa3ebNhA9RhK4eJC2ybWTG7i44GzZ62T3PzzD/D777RqY40X+wULqIB80SKaL2YtV65Q3dtbb1GtjbW3wphlcXLDmGSDB9OJlDFjgKVLZUejLGo1HQveuVN2JClUKuvPlzp3jmqzrJHcLFpEU8c//NDyj/Xvv/R7/9ln1N/IWqKjqeeUjw/18LHmWAlmHZzcMKYAX31F7e379QM2bpQdjXJUqUJ1JkrcmrJmcqPRUJF19eqWfZz4eGDJEjoSnSuXZR9LCFodypULmD7dso+V2vPnQLt29PPbto0GgjL7w8kNYwqgUgHz5lEDMbUa+OMP2REph1pNRZ9KOjYvI7mpWJFWVCxp/Xrg/n2aam9pa9cCv/0GzJ8PmHlsYYYSE+kU3sWLdGKxZEnrPC6zPk5uGFMIZ2fa+2/cmAb2nTghOyJlCAwE4uKow7NSyEhurNHfJjSUerxUrmzZx3nyBBgyhAp5O3Wy7GPpCEFJ286dNA6lZk3rPC6Tg5MbxhTEzY3ePZcvTzUBV6/Kjki+smXphV1JDf2smdwkJgKnTlm+3ub0aeDgQesc/x41iqaqz5tn+cfSmTKFttyWLKGTisy+cXLDmMLkzk1L5nny0B9hHq1GW1NbttBQQyWwZnJz8SLw6pXlk5uwMBpY2rGjZR9n3z4qnJ8xAyhc2LKPpbNsGTBpEvDNN1RPxOwfJzeMKVCBArR8npBAKzhPn8qOSK5u3egF3prTqTNjzeQmMpI++/tb7jFiY2lLtF8/y04cf/mSHqNxY+ptYw3bt9NjfvYZMG6cdR6TycfJDWMKVaIEDfC7fZtqcF6+lB2RPMWLU0M5pZyasmZyo9EA5coBlpzz+9NPlDxaOuH4+msaHrt4MZ3+srTISOCDD2iq+bx53KTPkXByw5iCValC2zGRkVRYq7Q5S9akVtNq1uPHsiOxfnJjyS0pIaiQuEMHmgJuKSdP0uymCROAChUs9zg6N24AbdvSv6Fff6W5XMxxcHLDmMLVr0/HZrdto3fWjjqmoWtXmlK9YYPsSKyX3Gi1dGrOksnNn3/SbCdLFhInJtLvbqVKwOjRlnscnYcPaTs3d27qtmzpI/RMeTi5YcwGtGkDrFhBH198ITsaOfz86JiyEramrJXcXLlCx+AtmdyEhtJKyrvvWu4xfviBVh+XLAFcXS33OABt377/Ph03376d6teY4+GFOsZsxIcfAo8eAZ9/Tn+wR42SHZH1qdXUyTk6GvD1lReHtZIbjYY+Wyq5uXePWg/MnGm5epQbN2grasgQoG5dyzyGTlIS0KMHHZ3ftw8oU8ayj8eUi1duGLMhQ4fS5OTRo2kVx9F07kyFqOvWyY3DmslNyZKWGxGwZAmdjrLU8WghKBnNn5+OYVuSEJT4b95MPZHq1LHs4zFl4+SGMRszZQodbf30U+UcjbaW/PmBFi3kb01ZM7mx1KpNYiKwcCGtCObJY5nH+OUXKgIPC6P6F0uaMYNGOSxYQLOjmGPj5IYxG6NSpZxuCQykglBHolYDBw7QRGlZdMmNJYu7hbBscvPbb8CdO5abI/XgATBsGNC9O51asqRffqHp4hMmWK9/DlM2Tm4Ys0HOzvQHvX59Kp48dUp2RNbToQONqVi9Wl4MHh6UfMTHW+4xbtyg5o2WSm5CQ4F69Sx3/8OH0/do7lzL3L/Onj1A795Ar17A5MmWfSxmOzi5YcxGubsDGzdS0WTr1sD167Ijsg5vbzo9JnNrysODPltya8qSxcSXLgG7d1vu+Pf27ZR8z54NFCxomccAaB5W58500mvRIm7Sx1JwcsOYDfPyov43uXPTHKqoKNkRWYdaDRw7Ji+hs1ZyU7iwZU6FLVhA9Utdu5r/vp8/pyLigACgZ0/z37/OrVuU5JYtC6xZY9mxEcz2cHLDmI0rWJCKNl+8oD/2MTGyI7K8du0owZA1KVzXFM7SyY0lVm3i4oDly4E+fWj1z9wmTgTu36cEylIrKU+e0O+6qyt18LZ0sTKzPZzcMGYHSpakOVQ3b1JNyqtXsiOyLE9PqjWStTVl6ZUbXTFxrVrmv+/wcBqU+dln5r/vo0eB77+nGVKW6jETH0+Ty6OiaNXSz88yj8NsGyc3jNmJatWo1fzRo3RCxd7nUKnVVHNx4YL1H9vSyc2dO3TayNwrN0LQcek2bYDSpc17369fU3sCf386JWUJWi1tdR09Sm0QKla0zOMw28fJDWN2pGFDqj/47Teqe7DnOVStW1PNkYytKUsnN5YqJj56lGZVWaKQeOZMmlG1eLHlhlSOGkW/36tW0e86Yxnh5IYxO9O2LdVULF0KjBsnOxrLcXMDOnWibRZrJ3GWTm4iI2nERpEi5r3f0FDawmzd2rz3e/kyHcMODrbc0fK5c+n01Q8/0M+dscxwcsOYHfr4Y3ohmDaNPtsrtZqONVu7z48uuYmLs8z964qJzVmQ+/AhrXL17099ksxFq6WO2UWKAF99Zb77TW3NGkqcRo0CBg+2zGMw+8KDMxmzU8OH06mVESMAHx/LHsuVpXlzOtIcHk61HtaSMyd9tuS2lLnnPS1fTitcn3xi3vtdtgz44w9g166UpM+cDhygZF2tpmSdMUPwyg1jdmzqVDry+8kndGTW3ri4AF260IqENbemnJzoGLUlkpuoKODuXfNu72i1NN+pWzfa7jKXe/doNaVXL+prY27nz9OpuAYNKDlz4lcsZiD+VWHMjqlU1G+kfXtq2PbXX7IjMj+1mo7AHz1q3ce11PDMEyfoszmTmx07aJyDuQuJhw6lXjMzZ5r3fgFK8Nq0AYoVAzZsoBorxgzFyQ1jdi5HDuDXX4E6daj53ZkzsiMyryZNqNeJtXveWCq50WhoSnepUua7z9BQ2rarV89897lpE7B2LfW1yZ/ffPcLUB+e996jFaetW2nkBmPG4OSGMQfg7k4vRiVLAq1a0bt4e+HsTNstERFAUpL1HteSyY05i4lv3qQtyYEDzXefMTF0f23b0mR6c0pIoK3GmzepSV/Roua9f+YYOLlhzEF4e9NAQw8PmkN1/77siMxHrab6j4MHrfeYlk5uzGXhQhpP0KOH+e5z7FhaXQkNNe+JLiGoEeCff9JQ2KpVzXffzLFwcsOYA/H1pTlUz59TPUNsrOyIzKNePaB4ces29LNEcvPoEa1YmCu5iY8Hliyhgl/dPKzs+usvKk6eOpW+5+b05ZfATz8BK1cC77xj3vtmjoWTG8YcTOnSVGB67RrN6LGHOVQqFW2PrFljvbETlkhuzF1MvHYt9bcZMMA89xcfTysrdeuavzg5LAwICaHiZLXavPfNHA8nN4w5oOrVaUTD4cPAhx9at1bFUtRqeiHfu9c6j2eJ5EajAXLlAsqVM8/9hYYC775rvhlMU6cCV6/SapA5GwFu2kTN+YYOpWZ9jGUXJzeMOajGjYHVq+mFZcAA259DVbMmJQXWOjVlqeTG3988/VxOngQOHTLfCsu5c7SyMnaseWthjhyhQa+dOlE3bXPW8DDHxckNYw6sfXt6F754MTBhguxoske3NbV+PW2fWJqlkhtzbUmFhQGFC1MTvOzSaoG+fWlL05zzyi5fpvYEtWoBP/9s3tUg5tg4uWHMwfXqBXz3HfDtt9SzxJap1XRMeedOyz+WuZOb2FjgyhV6oc+umBhKFvr1oy7O2RUWRluYixdTWwFziI6mAZ4FCtDqobnulzGAZ0sxxgCMHElHw4cNozlUH34oOyLTVKlCWybh4bQqZUnmTm5OnqTP5li5+d//aPWqb9/s39ft28CYMcBnn9FWpjk8f04rNi9fUo1UvnzmuV/GdDi5YYwBAKZPp4LcXr3oxaZNG9kRmUatptqQFy8sM8hRx9zJjUZDqxfZLf4VggqJO3Wibans3tfAgdQnZ/r07N2XTmIibR9evEhDMUuWNM/9MpYab0sxxgBQzcqiRdT2vksX2oawRYGBQFyc5QeFmju5iYwEatSgcRnZsX8/JQ7mKCReswb4/XdKlswxAkEIoH9/2jZcv966k9yZY+HkhjGWLEcO2tKpXZta6587Jzsi45UtS3Urlm7oZ4mVG3NsSYWGApUqZb8J3uPHwJAhQOfO1A/JHKZMAZYupY8WLcxzn4ylh5MbxpienDmBzZtpGnOrVsA//8iOyHhqNa3cWLIDs4cHzUEyR9PAuDhabclucnP3Lk3QNsccqVGjqG7nxx+zdz86y5YBkyYB33wD9OxpnvtkLCOc3DDG0siTh+ZQubnRHKoHD2RHZJxu3ajz8ubNlnsMXT3Py5fZv6/Tp+m4dXaTG91ppo8/zt797N1LyciMGdmv2wFoAGa/flSUbM6j5IxlhJMbxli6ChWi2oiYGKrDefZMdkSGK14caNjQsg39dMmNObamNBo6sl2liun38fo11Ux99FH26mNevqREpEkTGrWQXcePA1270u/QvHncpI9ZByc3jLEMlSlDKziXL9PpG2s0xzOXwEBKzh4/tsz9mzu5qVaNVspMtXkzbUtld47U5MnAv/9SopTdTsnXr1Ptlu54fnaLpRkzFCc3jLFM+fvTC+fBg7QqYCtzqLp2pVg3bLDM/Zs7ucnullRoKK1W1ahh+n2cOEGDKydMACpUyF48Dx9SOwEvL5pjZslj+Yy9iZMbxliWmjald97r19OAQ1uYQ+XnRyeGLLU1Za7kJj4eOHs2e8nNhQtUJ5Od49+JidT0r3JlKibOjhcvaOzDkye08legQPbujzFjcXLDGDNIx460VbFgAfDVV7KjMYxaTS/60dHmv29zJTdnzlBikZ3kZsECSiC6dDH9Pr7/nlaQFi8GXF1Nv5+kJOpwfeoUnVgrU8b0+2LMVJzcMMYM1qcPMG0a8PXX5jsibEmdO1PdyNq15r9vcyU3Gg0NjKxe3bTbx8UBK1ZQ8a+pNTvXr9NW1NChQN26pt0HQCt6Q4fSNubq1cDbb5t+X4xlByc3jDGjjB4NBAfTi9ivv8qOJnP581OzOEs09DNnclOpEvUXMsWqVXSS7bPPTLu9rmtwgQLUgyY7Zsyg2p8FC6iQmDFZuHadMWYUlYqmiD98SM3Y8uWjZn9KpVYDQUF0AqhoUfPdrzmTG1O3pHRzpNq1A0qUMO0+fvoJ2LUL2LoVyJXLtPsAgF9+oQGbEyaYZ2AnY9nBKzeMMaM5OQFLllBS07kz8PffsiPKWIcOtF2zerV579fVlb4P2UluXr+mBn6mJjdHjtA0cVMLie/fB4YPB7p3z96g1D17gN696WPyZNPvhzFzUURyM3/+fJQsWRLu7u6oW7cujh49muF1Fy9ejMaNGyNv3rzImzcvAgICMr0+Y8wyXFwoYahZkxq0XbggO6L0eXtTfOY+NaVSZX++1IULdFqqVi3Tbh8aCpQuTV2kTTF8OH2eO9e02wNUONypE/Duu8DChdykjymD9OQmIiICwcHBmDRpEjQaDWrUqIFWrVrh/v376V5///796N69O/bt24fDhw+jWLFiaNmyJe7cuWPlyBljHh7Uw6RwYXqBvXVLdkTpCwwEjh2jwllzym5yo9FQMmBKb5oHDyi5HDDAtGZ727ZRvc6cOUDBgsbfHqCf93vvAeXK0QRxFxfT7ocxc5Oe3MyePRt9+/ZF7969UblyZSxYsAAeHh5YtmxZutf/5ZdfMHDgQPj7+6NixYpYsmQJtFot9uzZY+XIGWMAkDcvsGMHdZ9t1YpqcZSmXTtKRMxdWGyO5KZ8eSB3buNvu2wZJUa9ext/2+fPqYi4RQvT51A9eUJbWa6udOTblOfAmKVITW4SEhIQGRmJgICA5MucnJwQEBCAw4cPG3QfL168wOvXr5EvXz5LhckYy0LhwjTq4NEjOiXz/LnsiPR5elJTOXNvTWU3uYmMNK3eJimJTiSp1XQizFgTJtDKz4IFpm0jvXpFfY+ioqhJn5+f8ffBmCVJTW4ePnyIpKQk+Pr66l3u6+uLqKgog+7jiy++QOHChfUSpNTi4+MRGxur98EYM79y5eiF7sIFKjJOSJAdkT61mop3z583331mJ7lJSqJiYFOSm+3bgZs3TSsk/vtvatj39ddUr2MsrZZOnx09SluS2R3TwJglSN+Wyo5p06YhPDwcGzZsgLu7e7rXCQkJgbe3d/JHsWLFrBwlY47jrbeATZuAP/6gY+JareyIUrRuTcXF5tyayk5yc/ky3daU5CY0lIqQjW2S9/o1HdOuWRMYNsz4xwVoNMOaNVSv06CBaffBmKVJTW58fHzg7OyM6Dd6o0dHR8Mvi3XOmTNnYtq0adi5cyeqZ9Lac+zYsYiJiUn+uH37tlliZ4ylr1kzau63Zg01+lPKHCo3N9pKiYgwX0zZSW40Gvpcs6Zxt7t+nYqBBw40fkvpu+9o5WrJEtMmdM+dC8yeDfzwA52QYkyppCY3rq6uqFWrll4xsK44uH79+hnebsaMGZgyZQq2b9+O2rVrZ/oYbm5u8PLy0vtgjFlW585UzzF/PjBliuxoUqjVwKVLdHzZHLKb3JQqRQXZxli4kFag1Grjbnf5Mm1FjRhhfEIFULIaHEwdqgcPNv72jFmT9A7FwcHBCAoKQu3atVGnTh3MnTsXcXFx6P3fEYCePXuiSJEiCAkJAQBMnz4dEydOxKpVq1CyZMnk2pxcuXIhV3baazLGzKpvXypa/fJLau0/YIDsiIDmzakANzwc8PfP/v15epp+OkyjMb6/zatXwNKldEJK1yHZEFot0K8fdWieNMm4xwSAP/8EPvqImv3996eYMUWTntwEBgbiwYMHmDhxIqKiouDv74/t27cnFxnfunULTqmaOISFhSEhIQEffPCB3v1MmjQJX9nKqGLGHMTYsdQFd9AgSiq6dZMbj4sL8MEHtDUVEpL9hnOmrtxotZTcjBlj3O3WrKETaf37G3e7pUupDmr3buOSIoC2sTp0ABo2pOPnpvTUYczaVEIoZUfcOmJjY+Ht7Y2YmBjeomLMCrRaKi5evZr6obRoITeeffuom+6RI9mbgA3QNs2OHcC5c8bd7to1oGxZOvVkzFyu+vWpn8zOnYbf5t49GszZuTMlJ8a4e5ce09sbOHCAPjMmizGv35yDM8YsyskJWL4cCAigItRjx+TG06QJ9WUxR88bU1duIiPpszG1LxoNJWTGHv8eMoSKqWfONO52sbHUpE+rpaGanNgwW8LJDWPM4lxcaEulenV6wbx4UV4szs60PRYRQb1mssPU5EajofoXY8YehIXRbdq1M/w2GzcC69bR6SZj+pwmJABdugD//EMns8w5TZ0xa+DkhjFmFZ6ewO+/A76+NIfq33/lxaJW03bNwYPZu5/sJDfG9Ld5+hT45Rfgs88MP8IdE0O1Tu3aGVfrJATw6adURLxxI1C1quG3ZUwpOLlhjFlNvnxUo6JSUYLz6JGcOOrVA0qUyH5DP11yY0zlohDGJzcrV1IDvk8/Nfw2Y8bQ1lJoqHGF019+Cfz0Ez3mO+8YfjvGlISTG0uaPx8oWRJwd6fKxaNHZUfEmHRFi1JB7IMHtKoQF2f9GFQqWs1YswZITDT9fjw8qCbFmFETt29TUmdociMEJShduhg+w+ngQeozFBICGNOUPSyMbjNzpvF9dBhTEk5uLCUigo5STJpEb9Nq1KBjEffvy46MMekqVKBajrNn6UVbxhwqtZp61Ozda/p96I5VG7M1petMbGiPm717qQGfoYXEr15Rj6F69YzrLbRpEzXn+/xz+tPFmC3j5MZSZs+mvzC9ewOVK9PbKA8P489iMmanatemmo59+4Bevaw/h6pmTRr2mZ1TU6YmN76+QKFChl0/NBSoUgVo3Niw60+dSkfNlyyh4mlDHDlCDfo6dQJmzcp+/x/GZOPkxhISEuisZ+pJ5U5O9P+HD8uLizGFad6cCmXDw2mQozW7bqlUtHqzfj0QH2/afZia3Lz1lmEJxL//0oqKoXOkzp4Fpk2jepsqVQyL5/Jl2h6sVQv4+WfDEyLGlIyTG0t4+JDOmP7XZTmZry/w37gIxhj54ANanfjxR1p1sKbAQDpVZExTvNRMSW4iIw2vt1m8GMiZk0YfZCUpiRaLy5ShomBDREfTtPSCBSmJcnc37HaMKZ308QuMMda/PxUYjx8P+PjQkWdrqFKFjjqHhwPt2xt/e11yY2hR9L179P7GkOTm9Wtg0SLg448BQ5qph4XR9tKBA9S0LyvPnwNt2wIvX9LWoDF9cBhTOk5uLMHHh9Z2o6P1L4+ONvy4A2MOZvx4qrcfMIDmUL0xPs5i1Go6IfTihfFzl4xdudEVExuS3GzcSImQIUXBt27RHK/+/YFGjbK+fmIirVpdukTJUIkSWd+GMVvC21KW4OpKG9h79qRcptXS/9evLy8uxhRMpQK+/56SjQ8/zN4pJmMEBtLKy5Ytxt/WlOQmb17DkonQUCoirlYt8+sJQTU5Xl5Ub5MVISgJ2rmT6o3MMR2dMaXh5MaMDh4Erl7979RHcDBtmK9cCVy4QG+/4uLo9BRjLF1OTsCKFUCzZjSJWjeDyZLKlqWTW6Y09DMluTGkmPj8eWD/fsOOf+sGks6fb9j8p6+/pinhS5fKH2LKmKVwcmNGgYF0tNTbG2j4QyBW15mJZ8EToa3hjyTNSRoB/GaRMWNMj6srzUOqUoXmUF2+bPnHDAykBCE21rjb5cxJn41JbgzpbxMWRkW+nTtnfr3Hj4GhQ6lXUMeOWd/vsmXAV18B335Lk9oZs1ec3JiRRkOt5SdMoMbEkx8NRp6n/8D5dTxcNX+jUq+6UKtp6XjbNioutObRV8ZshacnJRs+PjSm4c4dyz5et27U/G7zZuNu5+xMxbuGJDcPH1JtTFb1Ns+f04Jv376U6GVm5Eg6xv7jj1k//rZtQL9+VKw9dmzW12fMlnFBsRnpBgK2bJly2cuXwLlzwMmTwKlT9HnrVuDZM/p6gQK05+3vT02M/f2pe6uhw/EYs1f589ObhYYNqbn3n39a7kRP8eL0OOHhhh27Ts3Q4ZknTtDnrJKbX36hHex+/TK/3p49wPLldKIqq4aAx48DXbsC770HzJvHTfqY/eOXUAvLmZP282vXTrlMqwVu3kxJdk6don3z776jr7u50fFUXbJTowZ9GLKfzpg9KVaMCl8bNaKj2rt2GX+iyVBqNTB8OG31GJNEGZrcREYCuXNTH5qM6OZItW9PCVdGXrygFZimTYE+fTJ/3OvX6ci37sg7v3FijoB/zSVwcgJKl6aPTp1SLn/yBDh9OiXpOXmSOobq5u6ULKm/wlOjBl3G78KYPatYkVY7332XVh82bgRcXMz/OB98QHOVNmzIOmFIzdDkRqOhkQ9OmRQDHDpEfwN0b3QyMnkydS/eujXz+3v4kJr0eXkBv/1mucSQMaXh5EZB8uald2JNm6Zc9vo19aLQJTunTtGpiIcP6eve3kD16vpbW1WqcKdRZl/q1KGko21b4JNPqCYlsxd1U/j5Ae+8Q6sblkpusmoUGBpKp7dST25504kTNP/p66+B8uUzvt6LF/R4T5/S1JcCBbKOkTF7wcmNwrm40HJy1aoptQBCUDFy6jqenTtpL10IKnKsWDHtKk/BgvKeB2PZ1aIF8NNPNODRx4dm05p71VKtph4w0dGGH2w0JLl5+pSGWWZWb3P/PrBmDR04yChxS0wEPv2UZvGOGpXxfSUlUa+g06fpSHlmW2GM2SNObmyQSgUULkwf772XcnlcHA3OS530bNyY0hq+UCH9ZMffn46u86A8ZisCA4FHj4BBgyhZN/epn86dqbfM2rX0GIYwJLk5eZI+Z5bcLF1K/xZ79cr4OnPn0srNkSMZb80JQcfDf/uN5kW9/XbmsTFmjzi5sSOenkDduvSho9XSO8Y363h0nUxz5qQOqLpkx9+f/j93buvHz5ghBg6kVY5x42ir5dNPzXff+fPTaceICPMmNxoN/VurWDH9ryclAQsW0KpURsXM168DEydSXVCdOhk/1vTptL21aBFt4zHmiDi5sXNOTrQ6U66c/qyeR4/0T2v9/TcdK01MpK+XLZt2ladoUS5eZsowaRIN2vzsM0pIUhfmZ1dgIBAURAW7RYtmfX0Pj5QauIxoNPRvKKNV0q1bqQdORh2JhaDnWrAgMGVKxo/z88+0mjVhAvXJYcxRcXLjoPLnp9Mn776bcll8PE2KSJ30zJ5Np7gAKnh+s46ncuWsG40xZm4qFfDDD5RUdO9Ozb/fecc8992hA7VjWL2apqhkxdCVm9T/1t4UGkrbR6lbRqT2v/8Bu3dTEpQrV/rX2bOHiq1796bTVIw5MpUQjtUjNzY2Ft7e3oiJiYGXl5fscBRPCHoHm7qO5+RJ2uoCaN+/cuW0PXny55cXM3Mc8fFAu3a08vjHH3TU2hw6d6bf+6NHs77u4ME0V05XV/Om58/pKPaSJZR8vOnaNVopXb48/Xqb+/eBSpXoSPcvv6T/GKdO0ZDNhg2py7IljsozJpsxr9+8csMypVJRI7VixfSPsT57pt+T59QpOunx8iV9vWjRtKs8ZcqY//guc2xubjTZunlzevE/eJC2YLNLrabtqevXqR9VZrJauTl1it4kZFRMvGABrYoGBqb/9WHD6N/h3Lnpf/3WLTpYUK4c/RvkxIYxTm6YiXLnpneJDRumXJaUBFy5or/Ks2wZHVsHqOBZ15NHl/RUq8aNxVj25M5N2zWNGlEx8F9/0UnC7Gjbln4vIyKyPpGVVXKj0dDWbeXKab/28iX9G/nkk5QhnKlt3Qr8+iv19UmvT82TJzRc1NWVZnFltGXFmKPhbSlmcffv66/wnDwJXLxIyZBKRY3I3lzlKVSIi5eZcW7dAho0oNNGf/xBqyHZ0b07cP48/c5mZsYMOn34+HH6X+/dGzhzhuY7vWnlStqKunKFtqZSe/6cGnJWqEAztt789/DqFc3cOnuWOhtXqGDwU2PMJvG2FFOUggWpAVuLFimXvXpFA0VT1/Fs2wbExtLXCxRIe1qrQgVecmcZK15cfw7Vzp3ZWxVUq4GOHSnBSW/VRceQlZvU7RlSCw2lBOXNxAYAxo+ngun9+9MmNlotneg6epQKiTmxYUwfJzdMCnd3oFYt+tARIu1A0bVrgZkz6euurukPFM2Tx/rxM2WqXJm2cpo3pxqW9etNT4hbt6bxJhERmZ8+8vCgwuakpLRHvXVJfHpHvI8fp+Rk06a0X/v7bzoN9t13QKlSab8+ciTV16xbR6tVjDF9vC3FFO/p07QDRc+doxcUAChRQn+2lr8/DxR1dNu30+rNhx9STYuphey9e9NcpgsXMv59Cg+nLaxnz9LWvBw7Rg33jh5N2ym4Tx863n39un5SlJBASb+bG3UifnOK95w5dER93jzDGw0yZg94W4rZlTx5gCZN6ENHN1A09SpPaCg1dgPo6K1uZSf1QNH0ijaZ/WndmupZPvyQtjizmrKdkcBAYMUK+v3y90//Orqtrxcv0iY3Gg0lLtWq6V/+5AmwahU123tztee77yiZOn48bWKzZg0wYgQwejQnNoxlhpMbZpNSDxT98EO6TAggKkq/cHn3bpqirhsoWqFC2uJlQwckMtvSowfVrHz+OSU4o0cbfx/Nm1PPpvBww5KbN0VGUlLt7q5/+YoVtI315vTxS5do2vfIkWkf788/aXhujx5ASIjxz4UxR8LJDbMbKhWdsipUiI7H6rx4kXag6ObNdBoFAPz80iY85cvzQFF7MHQoreZ98QVNEk+viV5mXFxobElEBCUU6W1NZZbcaDRp+9totUBYGN1v6sRaqwX69aOeUpMm6d/m3DnqnNyoUfa22RhzFJzcMLvn4UF1D6mHDWq1VOuQuo5n1SoaOgjQ9lXVqvpJT/XqPFDUFn39NSU4ffvSKkyHDsbdXq0GFi6kupn0Tj1llNwkJNAR8KAg/cv37KGj38uW6V++ZAmtzuzZo799evcuJevFilGBNI87YSxrnNwwh+TkRMdvy5YFunRJufzxY/06nqNHaQvh9Wv6epkyaY+oFyvGxctKplLR1uTDh1RDs3Onfv1WVho3ptW98HDjkpvz5ynBeXPlJjSUanBSN8C8excYNYpWllLPoIqNpcRGCDoF5u1teNyMOTJObhhLJV8+oFkz+tBJSEg7UHTu3JSmbXnzpk14eKCosjg701ym996jU1R//kk/K0Nv260bbU3NnJl2uzKj5EajocQq9ePcvk1bovPn6yfEQ4ZQXU7qwueEBJpx9c8/1HXZkAnljDHCyQ1jWXB1TTl51bMnXaYbKJo64fn995T5PzlypD9Q1MdH0pNgcHMDNm6kxLVVK0oYypQx7LZqNfWdOXgQaNpU/2uZJTcVK+qfoFq0iMaQ6IrgAWDDBtpuioig5Bqg368+fYADB2ilqUoVo54qYw6PkxvGTJB6oGi7dimXP3tGdRZvNiLUDRQtUkR/hcffnweKWpNuDlXjxilzqPz8sr5dvXrUTyk83LjkJvWWVEICsHgxJci62q2nT+lId7t2QNeuKdf98kvg55/TfzzGWNY4uWHMjHLnpo6xqbvGJiUBV6/qn9ZasYLqLICUgaKpV3mqVaPLmfkVLEizmho2pH44+/dn3eVapaJ6nWXLgB9/1O8/4+ZGyWnq5CYpiX7OH3yQctmGDUB0NDBgQMplY8ZQQhwamrJNFRZGJ7Nmzsx4UjhjLHOc3DBmYbr+OhUq6L9YPXigv8Jz8CC9s9cNFC1XLu0R9cKFuXjZHEqWpASnSRM6PbV9e9YNHgMDaUjm3r206qOjUqWdL3XxIq3WpV65CQ2lVRjdFtOBA3QKa948WgEEaBTD4MHUmyc42CxPlTGHxMkNY5IUKAAEBNCHzqtXdMom9SrPjh1ATAx93ccnbcJTsSIPFDVF1apUJxUQQDU169al7QicWs2alHCGh+snN0Da5Eajoc+6Rnxnz1IRc0QE/f+rV3Q0vX79lJWcw4cpjs6dgdmzOYllLDs4uWFMQdzd6d1+6nf8QtCJmdSrPOvXA7Nm0dddXWk14M2khweKZq1BA6qJev99aqC3dGnGSYVKlVJYHBZG21E66SU3Zcqk/AzCwqi2p2NH+v9vv6U+S+vW0ZbW5ct0iqt2beCnn7gGi7Hs4uSGMYVTqWgbpWRJ/QZ0MTH6A0VPnaJGhKkHir55RL1kSX7hfNN771EN1McfUz3OtGkZX1etBqZMoRNM7dunXJ5ecqNLUJ89A/73P2D4cEpEz5yhxxg3jpLS6Giq/SlYkLal3hzVwBgzHic3jNkob2869dO4ccpliYm0CqDrunzqFLBgAXD/Pn09d+60A0WrVuWBoh99RE3+hg+n7cIRI9K/XuXK9P0KD884udFqgRMnKHkB6NTTy5e0MpSURNtRZcvS158/B9q2pW2qfftSjoIzxrKHkxvG7Iiuv07lyjRgUefNgaJ799JWiVZLKzkVK6Zd5XG0gaLDhlESOHIk1Ta9OTZBR62m00wvXqQcA0+d3Fy7Rqs1tWrRlmJoKG17FS1KJ63+/puKiZ2cqDng5ctUj1OihFWeJmMOgZMbxhyAnx9tfbRunXLZixc0kDF10vP77/TCDFByk95A0cyKbm3dt9/SKbY+fWgVJfXqjE5gIDB+PLBlS0pvmtTJja6YuGZNOgF39iwwZw5w6xYwdiwVEDdsSCs4u3ZR352MJo4zxkxjx3+mGGOZ8fAA3n6bPnS0WuDGDf2EJzycjkADVA+iGyiqS3iqVwe8vKwfvyWoVLSi9egRrars2kWTuFMrW5YKfyMiMk5uihen1Z/QUEoImzWj1Zs8eWjV5+uvqXh55UqgRQurPkXGHIJKCCFkB2FNsbGx8Pb2RkxMDLzs5S8yYxb2+DEVL6dOes6dSxkoWrp02lWe4sVt9zjzq1c0sPLECdoyql5d/+uzZtHqTXQ0JXYff0xzo/bvp6PluXNTklS8OCWGfn5A9+40/uHhQ+DTT2mVSFeXwxjLmjGv35zcMMZMkpBAzepSn9Y6eZJWPQBapUhvoGjqI9RKFhsLvPMOcO8ecOgQUKpUytdu36bE5aefqBj5s88oEfr7byB/fipMdnKiBObMGepn07Qp0Ls3reD07avflZgxljVObjLByQ1jliMEjZVIneycOgVcuUJfy5EDqFQp7SqPUgeKRkenbEsdPKhfZN2oESVwv/9OycyuXfTfpUqldBpu2ZJOSG3YQNt7XbrQys769WmnizPGMmfM6zfX3DDGzEalouGgRYrQEWed58/TDhRdty6lTqVIkbSrPGXKyE8AfH2pp02DBrRNtW8fHcEH6NTU8OG0ZaerudEVEz96RKs7tWoBAwcCU6fS6atq1YBff5X/vBizd7xywxiTIimJjk2/ucpz5w593cMj/YGiuXJZP9bTp2kOVc2awLZtVFgdFUVJ2aJF9N8//ki1NEuXUtyPH9OHnx+dwBKCtrcKFLB+/IzZA96WygQnN4wp28OHKcmOLuG5cIEaFKpUdFop9QqPv791BooePEgnm9q0AVavpi225s2ptua994BJk2ir6uVLKix+7z1g926qM7p7lxKbMmUsGyNj9oyTm0xwcsOY7YmPTztQ9NQp4OlT+nr+/GnreCpVMv9A0d9+Azp1osLgRYuAJUuA/v1pnMKYMVQ7VLo0xfrsGTVH/OcfSnZSH7lnjBmPa24YY3bFzY22hGrWTLlMCGqMlzrZ2biRJmoDNMepcuW0SU/evKbH0b49bTv16pUypmHgQGrUp9VSh+O4ONpSc3Gh7sObNnFiw5i1cXLDGLNJKhWNLChRgo5X68TGph0oGh5OvWsAOsL9ZsJTqpThA0WDgmjrbORISnBatgSOHUv5elwcFRcLASxerF9YzRizDkXMB54/fz5KliwJd3d31K1bF0ePHs30+mvWrEHFihXh7u6OatWqYevWrVaKlDGmdF5eVPsyaBAlF0eP0hbRuXM0NV2tpm2uRYvoaHbZsnSku1EjOr69eDElKy9fZvwYI0YAo0cDwcFAsWJUEwRQgqRSUWIzcSIVGDPGrE96zU1ERAR69uyJBQsWoG7dupg7dy7WrFmDS5cuoWDBgmmuf+jQITRp0gQhISFo164dVq1ahenTp0Oj0aBq1apZPh7X3DDGdKKiaGUn9SrPxYspA0UrVEh7RN3Pj24rBCUvK1fS/yclpdzvRx8B//sfN+ljzJxsqqC4bt26ePvttzFv3jwAgFarRbFixTBkyBCMGTMmzfUDAwMRFxeH33//PfmyevXqwd/fHwsWLMjy8Ti5YYxl5uXLtANFT53SHyiqS3SqVgVWrKD+N7q/pLVqAYcPm7+YmTFHZzMFxQkJCYiMjMTYsWOTL3NyckJAQAAOHz6c7m0OHz6M4OBgvctatWqFjRs3pnv9+Ph4xMfHJ/9/bGxs9gNnjNmtnDlpMGbt2imXabXAzZv6CU9ERMpAUR0nJzoZxYkNY3JJTW4ePnyIpKQk+KbuaQ7A19cXFy9eTPc2UVFR6V4/Kioq3euHhIRg8uTJ5gmYMeaQnJzoiHfp0kDnzimXP3lCxcvTp1Nzv+7d5TQZZIzpU0RBsSWNHTsWMTExyR+3b9+WHRJjzE7kzUsDMbdupW2pn3+WHRFjDJC8cuPj4wNnZ2dER0frXR4dHQ0/XdXeG/z8/Iy6vpubG9xsZQwxY4wxxrJN6sqNq6sratWqhT179iRfptVqsWfPHtSvXz/d29SvX1/v+gCwa9euDK/PGGOMMccivYlfcHAwgoKCULt2bdSpUwdz585FXFwcevfuDQDo2bMnihQpgpCQEADA559/jqZNm2LWrFlo27YtwsPDcfz4cSxatEjm02CMMcaYQkhPbgIDA/HgwQNMnDgRUVFR8Pf3x/bt25OLhm/dugWnVK1DGzRogFWrVmH8+PEYN24cypUrh40bNxrU44Yxxhhj9k96nxtr4z43jDHGmO0x5vXb7k9LMcYYY8yxcHLDGGOMMbvCyQ1jjDHG7AonN4wxxhizK5zcMMYYY8yucHLDGGOMMbvCyQ1jjDHG7AonN4wxxhizK5zcMMYYY8yuSB+/YG26hsyxsbGSI2GMMcaYoXSv24YMVnC45ObZs2cAgGLFikmOhDHGGGPGevbsGby9vTO9jsPNltJqtbh79y5y584NlUqV7fuLjY1FsWLFcPv2bYedVeXo3wNHf/4Afw8A/h44+vMH+Htg6ecvhMCzZ89QuHBhvYHa6XG4lRsnJycULVrU7Pfr5eXlkL/MqTn698DRnz/A3wOAvweO/vwB/h5Y8vlntWKjwwXFjDHGGLMrnNwwxhhjzK5wcpNNbm5umDRpEtzc3GSHIo2jfw8c/fkD/D0A+Hvg6M8f4O+Bkp6/wxUUM8YYY8y+8coNY4wxxuwKJzeMMcYYsyuc3DDGGGPMrnBywxhjjDG7wsmNAb799ls0aNAAHh4eyJMnj0G3EUJg4sSJKFSoEHLmzImAgABcuXJF7zqPHz/Ghx9+CC8vL+TJkwd9+vTB8+fPLfAMssfYOG/evAmVSpXux5o1a5Kvl97Xw8PDrfGUjGbKz+qdd95J8/z69++vd51bt26hbdu28PDwQMGCBTFq1CgkJiZa8qmYxNjn//jxYwwZMgQVKlRAzpw5Ubx4cQwdOhQxMTF611Py78D8+fNRsmRJuLu7o27dujh69Gim11+zZg0qVqwId3d3VKtWDVu3btX7uiF/E5TGmO/B4sWL0bhxY+TNmxd58+ZFQEBAmuv36tUrzc+7devWln4aJjPm+a9YsSLNc3N3d9e7jr3/DqT3N0+lUqFt27bJ17Ha74BgWZo4caKYPXu2CA4OFt7e3gbdZtq0acLb21ts3LhRnDp1Srz//vuiVKlS4uXLl8nXad26tahRo4Y4cuSIOHDggChbtqzo3r27hZ6F6YyNMzExUdy7d0/vY/LkySJXrlzi2bNnydcDIJYvX653vdTfHyUx5WfVtGlT0bdvX73nFxMTk/z1xMREUbVqVREQECBOnDghtm7dKnx8fMTYsWMt/XSMZuzzP3PmjOjcubPYvHmzuHr1qtizZ48oV66c6NKli971lPo7EB4eLlxdXcWyZcvEuXPnRN++fUWePHlEdHR0utf/66+/hLOzs5gxY4Y4f/68GD9+vHBxcRFnzpxJvo4hfxOUxNjvQY8ePcT8+fPFiRMnxIULF0SvXr2Et7e3+Pfff5OvExQUJFq3bq338378+LG1npJRjH3+y5cvF15eXnrPLSoqSu869v478OjRI73nf/bsWeHs7CyWL1+efB1r/Q5wcmOE5cuXG5TcaLVa4efnJ7777rvky54+fSrc3NzEr7/+KoQQ4vz58wKAOHbsWPJ1tm3bJlQqlbhz547ZYzeVueL09/cXn3zyid5lAMSGDRvMFarFmPo9aNq0qfj8888z/PrWrVuFk5OT3h/AsLAw4eXlJeLj480SuzmY63dg9erVwtXVVbx+/Tr5MqX+DtSpU0cMGjQo+f+TkpJE4cKFRUhISLrX79atm2jbtq3eZXXr1hWfffaZEMKwvwlKY+z34E2JiYkid+7cYuXKlcmXBQUFiQ4dOpg7VIsw9vln9frgiL8Dc+bMEblz5xbPnz9PvsxavwO8LWUBN27cQFRUFAICApIv8/b2Rt26dXH48GEAwOHDh5EnTx7Url07+ToBAQFwcnLC33//bfWYM2KOOCMjI3Hy5En06dMnzdcGDRoEHx8f1KlTB8uWLTNolL21Zed78Msvv8DHxwdVq1bF2LFj8eLFC737rVatGnx9fZMva9WqFWJjY3Hu3DnzPxETmet3NSYmBl5eXsiRQ3+kndJ+BxISEhAZGan379fJyQkBAQHJ/37fdPjwYb3rA/Sz1F3fkL8JSmLK9+BNL168wOvXr5EvXz69y/fv34+CBQuiQoUKGDBgAB49emTW2M3B1Of//PlzlChRAsWKFUOHDh30/h074u/A0qVLoVar4enpqXe5NX4HHG5wpjVERUUBgN6Llu7/dV+LiopCwYIF9b6eI0cO5MuXL/k6SmCOOJcuXYpKlSqhQYMGepd//fXXePfdd+Hh4YGdO3di4MCBeP78OYYOHWq2+M3B1O9Bjx49UKJECRQuXBinT5/GF198gUuXLmH9+vXJ95ve74jua0phjt+Bhw8fYsqUKejXr5/e5Ur8HXj48CGSkpLS/dlcvHgx3dtk9LNM/e9dd1lG11ESU74Hb/riiy9QuHBhvRfH1q1bo3PnzihVqhSuXbuGcePGoU2bNjh8+DCcnZ3N+hyyw5TnX6FCBSxbtgzVq1dHTEwMZs6ciQYNGuDcuXMoWrSow/0OHD16FGfPnsXSpUv1LrfW74DDJjdjxozB9OnTM73OhQsXULFiRStFZF2GPv/sevnyJVatWoUJEyak+Vrqy2rWrIm4uDh89913Vnths/T3IPULebVq1VCoUCE0b94c165dQ5kyZUy+X3Ox1u9AbGws2rZti8qVK+Orr77S+5rs3wFmGdOmTUN4eDj279+vV1SrVquT/7tatWqoXr06ypQpg/3796N58+YyQjWb+vXro379+sn/36BBA1SqVAkLFy7ElClTJEYmx9KlS1GtWjXUqVNH73Jr/Q44bHIzYsQI9OrVK9PrlC5d2qT79vPzAwBER0ejUKFCyZdHR0fD398/+Tr379/Xu11iYiIeP36cfHtLMvT5ZzfOtWvX4sWLF+jZs2eW161bty6mTJmC+Ph4q8wmsdb3QKdu3boAgKtXr6JMmTLw8/NLc/IgOjoaAOzmd+DZs2do3bo1cufOjQ0bNsDFxSXT61v7dyA9Pj4+cHZ2Tv5Z6ERHR2f4fP38/DK9viF/E5TElO+BzsyZMzFt2jTs3r0b1atXz/S6pUuXho+PD65evaqo5CY7z1/HxcUFNWvWxNWrVwE41u9AXFwcwsPD8fXXX2f5OBb7HbB4VY8dMbageObMmcmXxcTEpFtQfPz48eTr7NixQ7EFxabG2bRp0zQnZDLyzTffiLx585ocq6WY62d18OBBAUCcOnVKCJFSUJz65MHChQuFl5eXePXqlfmeQDaZ+vxjYmJEvXr1RNOmTUVcXJxBj6WU34E6deqIwYMHJ/9/UlKSKFKkSKYFxe3atdO7rH79+mkKijP7m6A0xn4PhBBi+vTpwsvLSxw+fNigx7h9+7ZQqVRi06ZN2Y7X3Ex5/qklJiaKChUqiOHDhwshHOd3QAh6rXRzcxMPHz7M8jEs9TvAyY0B/vnnH3HixInk48wnTpwQJ06c0DvWXKFCBbF+/frk/582bZrIkyeP2LRpkzh9+rTo0KFDukfBa9asKf7++29x8OBBUa5cOcUeBc8szn///VdUqFBB/P3333q3u3LlilCpVGLbtm1p7nPz5s1i8eLF4syZM+LKlSsiNDRUeHh4iIkTJ1r8+ZjC2O/B1atXxddffy2OHz8ubty4ITZt2iRKly4tmjRpknwb3VHwli1bipMnT4rt27eLAgUKKPYouDHPPyYmRtStW1dUq1ZNXL16Ve/YZ2JiohBC2b8D4eHhws3NTaxYsUKcP39e9OvXT+TJkyf5ZNvHH38sxowZk3z9v/76S+TIkUPMnDlTXLhwQUyaNCndo+BZ/U1QEmO/B9OmTROurq5i7dq1ej9v3d/JZ8+eiZEjR4rDhw+LGzduiN27d4u33npLlCtXTlHJvI6xz3/y5Mlix44d4tq1ayIyMlKo1Wrh7u4uzp07l3wde/8d0GnUqJEIDAxMc7k1fwc4uTFAUFCQAJDmY9++fcnXwX/9OnS0Wq2YMGGC8PX1FW5ubqJ58+bi0qVLevf76NEj0b17d5ErVy7h5eUlevfurZcwKUVWcd64cSPN90MIIcaOHSuKFSsmkpKS0tzntm3bhL+/v8iVK5fw9PQUNWrUEAsWLEj3ukpg7Pfg1q1bokmTJiJfvnzCzc1NlC1bVowaNUqvz40QQty8eVO0adNG5MyZU/j4+IgRI0boHZVWCmOf/759+9L9NwNA3LhxQwih/N+BH3/8URQvXly4urqKOnXqiCNHjiR/rWnTpiIoKEjv+qtXrxbly5cXrq6uokqVKmLLli16Xzfkb4LSGPM9KFGiRLo/70mTJgkhhHjx4oVo2bKlKFCggHBxcRElSpQQffv2TdMLRkmMef7Dhg1Lvq6vr6947733hEaj0bs/e/8dEEKIixcvCgBi586dae7Lmr8DKiEUePaWMcYYY8xE3OeGMcYYY3aFkxvGGGOM2RVObhhjjDFmVzi5YYwxxphd4eSGMcYYY3aFkxvGGGOM2RVObhhjjDFmVzi5YYw5tF69eqFjx46yw2CMmREnN4wxxerVqxdUKhVUKhVcXFxQqlQpjB49Gq9evZIdGmNMwRx2KjhjzDa0bt0ay5cvx+vXrxEZGYmgoCCoVCpMnz5ddmiMMYXilRvGmKK5ubnBz88PxYoVQ8eOHREQEIBdu3YBALRaLUJCQlCqVCnkzJkTNWrUwNq1a5Nvm5SUhD59+iR/vUKFCvj+++9lPRXGmJXwyg1jzGacPXsWhw4dQokSJQAAISEh+Pnnn7FgwQKUK1cOf/75Jz766CMUKFAATZs2hVarRdGiRbFmzRrkz58fhw4dQr9+/VCoUCF069ZN8rNhjFkKJzeMMUX7/fffkStXLiQmJiI+Ph5OTk6YN28e4uPjMXXqVOzevRv169cHAJQuXRoHDx7EwoUL0bRpU7i4uGDy5MnJ91WqVCkcPnwYq1ev5uSGMTvGyQ1jTNGaNWuGsLAwxMXFYc6cOciRIwe6dOmCc+fO4cWLF2jRooXe9RMSElCzZs3k/58/fz6WLVuGW7du4eXLl0hISIC/v7+VnwVjzJo4uWGMKZqnpyfKli0LAFi2bBlq1KiBpUuXomrVqgCALVu2oEiRInq3cXNzAwCEh4dj5MiRmDVrFurXr4/cuXPju+++w99//23dJ8EYsypObhhjNsPJyQnjxo1DcHAwLl++DDc3N9y6dQtNmzZN9/p//fUXGjRogIEDByZfdu3aNWuFyxiThE9LMcZsSteuXeHs7IyFCxdi5MiRGD58OFauXIlr165Bo9Hgxx9/xMqVKwEA5cqVw/Hjx7Fjxw5cvnwZEyZMwLFjxyQ/A8aYpfHKDWPMpuTIkQODBw/GjBkzcOPGDRQoUAAhISG4fv068uTJg7feegvjxo0DAHz22Wc4ceIEAgMDoVKp0L17dwwcOBDbtm2T/CwYY5akEkII2UEwxhhjjJkLb0sxxhhjzK5wcsMYY4wxu8LJDWOMMcbsCic3jDHGGLMrnNwwxhhjzK5wcsMYY4wxu8LJDWOMMcbsCic3jDHGGLMrnNwwxhhjzK5wcsMYY4wxu8LJDWOMMcbsCic3jDHGGLMr/we2XnC0x//nWQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "freqs_for_each_token = torch.outer(torch.arange(len(tokens)), freqs)\n",
    "print(freqs_for_each_token.shape)\n",
    "freqs_cis = torch.polar(torch.ones_like(freqs_for_each_token), freqs_for_each_token)\n",
    "print(freqs_cis.shape)\n",
    "\n",
    "# viewing tjhe third row of freqs_cis\n",
    "value = freqs_cis[3]\n",
    "plt.figure()\n",
    "for i, element in enumerate(value[:len(tokens)]):\n",
    "    plt.plot([0, element.real], [0, element.imag], color='blue', linewidth=1, label=f\"Index: {i}\")\n",
    "    plt.annotate(f\"{i}\", xy=(element.real, element.imag), color='red')\n",
    "plt.xlabel('Real')\n",
    "plt.ylabel('Imaginary')\n",
    "plt.title('Plot of one row of freqs_cis')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### now that we have a complex number (the angle change vector) for every token's query element\n",
    "we can convert our queries (the one we split into pairs) as complex numbers and then dot product to rotate the query based on the position\n",
    "<br>\n",
    "honeslty this is beautiful to think about :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)\n",
    "q_per_token_as_complex_numbers.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token_as_complex_numbers_rotated = q_per_token_as_complex_numbers * freqs_cis\n",
    "q_per_token_as_complex_numbers_rotated.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### after rotated vector is obtained\n",
    "we can get back our the queries as pairs by viewing the complex numbers as real numbers again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64, 2])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers_rotated)\n",
    "q_per_token_split_into_pairs_rotated.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the rotated pairs are now merged, we now have a new query vector (rotated query vector) that is of the shape [7x128] where 7 is the number of tokens and the 128 is the dim of the query vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)\n",
    "q_per_token_rotated.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# keys (almost the same as queries)\n",
    "im lazy as fuck, so im not going to go through the math for keys, the only things you need to keep in mind are:\n",
    "<br>\n",
    "&gt; keys generate key vectors also of dimention 128\n",
    "<br>\n",
    "&gt; the number of keys equal to number of queries, qwen1.5-4B use MHA, instead of GQA\n",
    "<br>\n",
    "&gt; keys are also rotated to add positional info, just like queries because of the same reasons "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 128, 4096])\n",
      "torch.Size([2, 128])\n"
     ]
    }
   ],
   "source": [
    "#k_layer0 = model[\"model.layers.0.self_attn.k_proj.weight\"]\n",
    "#k_layer0 = k_layer0.view(n_kv_heads, 2, k_layer0.shape[0] // n_kv_heads//2, dim).permute(0,2,1,3).reshape(n_kv_heads, k_layer0.shape[0]// n_kv_heads, dim)\n",
    "k_layer0 = k_layer0.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "print(k_layer0.shape)\n",
    "\n",
    "#k_layer0_bias = model[\"model.layers.0.self_attn.k_proj.bias\"]\n",
    "k_layer0_bias = k_layer0_bias.reshape(n_kv_heads, -1)\n",
    "print(k_layer0_bias.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 4096])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_layer0_head0 = k_layer0[0]\n",
    "k_layer0_head0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_layer0_bias_head0 = k_layer0_bias[0]\n",
    "k_layer0_bias_head0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_per_token = torch.matmul(token_embeddings, k_layer0_head0.T.float()) + k_layer0_bias_head0.float()\n",
    "k_per_token.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64, 2])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)\n",
    "k_per_token_split_into_pairs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)\n",
    "k_per_token_as_complex_numbers.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 64, 2])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis)\n",
    "k_per_token_split_into_pairs_rotated.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)\n",
    "k_per_token_rotated.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## at this stage now have both the rotated values of queries and keys, for each token. \n",
    "each of the queries and keys are now of shape [7x128]. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## in the next step we will multiply the queries and key matrices\n",
    "doing this will give us a score mapping each token with one another\n",
    "<br>\n",
    "this score describes how well each token's query relates to the each tokens's key. \n",
    "THIS IS SELF ATTENTION :)\n",
    "<br>\n",
    "the shape of the attention score matrix (qk_per_token) is [7x7] where 7 is the number of tokens in the prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 8])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qk_per_token = torch.matmul(q_per_token_rotated, k_per_token_rotated.T)/(head_dim)**0.5\n",
    "qk_per_token.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# we now have to mask query key scores\n",
    "during the training process of qwen, the future token qk scores are masked.\n",
    "<br>\n",
    "why? because during training we only learn to predict tokens using past tokens.\n",
    "<br>\n",
    "as a result, during inference we set the future tokens to zero."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABBtUlEQVR4nO3deVxU9f4/8NdhGxAYEFyAZIlN0QQh1NzRMKWi3EK9Xdeye9NSc81fbmhJ4lKa5pZpdbt6M9G8ai6ZiFouqKAVkhAIlUZpioMBOvP5/eF1vo6IMnOGmcPwej4en8eDs33O++Agb96fzzlHEkIIEBEREVmInbUDICIiovqFyQcRERFZFJMPIiIisigmH0RERGRRTD6IiIjIoph8EBERkUUx+SAiIiKLYvJBREREFuVg7QDqG51Oh19//RXu7u6QJMna4RARkZGEELh27Rr8/PxgZ1d7f8OXl5ejsrJSdj9OTk5wdnY2Q0Tmw+TDwn799Vf4+/tbOwwiIpKpuLgYzZo1q5W+y8vL8XCgGy6WaGX35ePjg4KCAkUlIEw+LMzd3R0A0LbHNDg4KOeDUBNaVd0cpXP5rdzaIZhME+Bi7RBM0vCbYmuHYBrHOvpfYrn8v46t5c/OgdYOwWjaG+XI2vam/v/z2lBZWYmLJVoUnAiE2t30/3tLr+nw8KPnUVlZyeSjPrs91OLg4AwHR+V8EGpCcqybyYdDHf6U17XPyG0Odiprh2Aauzr6YambP5oA6u5nHIBFhs7V7naykg+lqqM/aURERLZPK3TQynj9q1bozBeMGTH5ICIiUigdBHQwPfuQc2xtYvJBRESkUDroIKd2Ie/o2mN7A0lERESkaKx8EBERKZRWCGiF6UMnco6tTUw+iIiIFMpW53xw2IWIiIgsipUPIiIihdJBQGuDlQ8mH0RERArFYRciIiKyaRkZGUhMTISfnx8kScLWrVsNtkuSdM+2YMECo87D5IOIiEihbt/tIqcZo6ysDFFRUVi+fPk9t1+4cMGgffjhh5AkCf379zfqPBx2ISIiUijd/5qc442RkJCAhISEarf7+PgYLH/xxRfo3r07goODjToPkw8iIiIy2m+//YYdO3bgo48+MvpYJh9EREQKpZV5t8vtY0tLSw3Wq1QqqFTy3j790Ucfwd3dHf369TP6WM75ICIiUiitkN8AwN/fHx4eHvqWkpIiO7YPP/wQzz//PJydnY0+lpUPIiIihTLXnI/i4mKo1Wr9erlVj4MHDyI3Nxf/+c9/TDqeyQcREZGNU6vVBsmHXGvXrsWjjz6KqKgok45n8kFERKRQOkjQQpJ1vDE0Gg3y8vL0ywUFBcjKyoKXlxcCAgIA3Jo/smnTJixatMjkuJh8EBERKZRO3GpyjjdGZmYmunfvrl+eMGECAGDYsGFYv349AGDjxo0QQmDw4MEmx8Xko5ZVVFSgoqJCv3z3jGMiIiKliIuLg3jAg8leeuklvPTSS7LOw7tdallKSorBDGN/f39rh0RERHWE9n/DLnKaEjH5qGXTpk3D1atX9a24uNjaIRERUR1hq8kHh11qmTke5EJERGRLmHwQEREplE5I0AkZd7vIOLY2MfkgIiJSKLlDJ0odduGcDyIiIrIoVj6IiIgUSgs7aGXUCbRmjMWcmHwQEREplJA550NwzgcREREZg3M+iIiIiMyAlQ8iIiKF0go7aIWMOR8y3gtTm5h8EBERKZQOEnQyBil0UGb2wWEXIiIisihWPoiIiBTKViecMvkgIiJSKPlzPjjsQkRERMTKBxERkVLdmnAq48VyHHYhIiIiY+hkPl6dd7sQERERgZUPIiIixbLVCadMPoiIiBRKBzubfMgYkw8iIiKF0goJWhlvppVzbG1i8mElDX76Ew72KmuHYZTKhzysHYJJhL0yf/hqwq3oL2uHYJKbv/xq7RBM4uDrY+0QTKIru27tEEzm9nO5tUMw2s2bdS9mpWHyQUREpFBamXe7aDnsQkRERMbQCTvoZEw41Sl0wilvtSUiIiKLYuWDiIhIoTjsQkRERBalg7w7VnTmC8WsOOxCREREFsXKBxERkULJf8iYMmsMTD6IiIgUSv7j1ZWZfCgzKiIiIrJZrHwQEREplA4SdJAz4VSZT3hm5YOIiEihbg+7yGnGyMjIQGJiIvz8/CBJErZu3Vpln5ycHDzzzDPw8PCAq6sr2rZti6KiIqPOw+SDiIhIoW4/50NOM0ZZWRmioqKwfPnye27Pz89H586d0aJFC6Snp+P06dOYMWMGnJ2djToPh12IiIgIAJCQkICEhIRqt7/xxht48sknkZqaql8XEhJi9HlY+SAiIlIonZBkN7PFotNhx44dCA8PR69evdCkSRO0b9/+nkMzD8Lkg4iISKF0Modcbj/no7S01KBVVFQYHUtJSQk0Gg3efvtt9O7dG3v27EHfvn3Rr18/HDhwwKi+mHwQERHZOH9/f3h4eOhbSkqK0X3odLce1v7ss8/itddeQ5s2bfD666/j6aefxsqVK43qi3M+iIiIFEon7KCT8aCw28cWFxdDrVbr16tUKqP7atSoERwcHNCyZUuD9RERETh06JBRfTH5ICIiUigtJGhlPKvj9rFqtdog+TCFk5MT2rZti9zcXIP1P/74IwIDA43qi8kHERERAQA0Gg3y8vL0ywUFBcjKyoKXlxcCAgIwefJkDBw4EF27dkX37t2xa9cu/Pe//0V6erpR56k3cz4uXryIcePGITQ0FM7OzmjatCk6deqEFStW4Pr169YOj4iIqIrbwy5ymjEyMzMRHR2N6OhoAMCECRMQHR2NmTNnAgD69u2LlStXIjU1Fa1bt8YHH3yAzZs3o3Pnzkadp15UPn766Sd06tQJnp6emDdvHlq3bg2VSoUzZ85g9erVeOihh/DMM89YO0wiIiIDWkDmsItx4uLiIIS47z4jR47EyJEjTY4JqCeVj9GjR8PBwQGZmZlISkpCREQEgoOD8eyzz2LHjh1ITEwEAFy5cgUvvvgiGjduDLVajR49eiA7O1vfz+zZs9GmTRt88sknCAoKgoeHBwYNGoRr165Z69KIiIjqHJtPPi5duoQ9e/ZgzJgxcHV1vec+knQrq3zuuedQUlKCL7/8EidOnEBMTAwef/xxXL58Wb9vfn4+tm7diu3bt2P79u04cOAA3n777WrPX1FRUeX+aiIiopqw9LCLpSgzKjPKy8uDEALNmzc3WN+oUSO4ubnBzc0NU6dOxaFDh3Ds2DFs2rQJsbGxCAsLw8KFC+Hp6YnPP/9cf5xOp8P69evxyCOPoEuXLhgyZAj27dtX7flTUlIM7q329/evtWslIiLbYukXy1mKMqOygGPHjiErKwutWrVCRUUFsrOzodFo4O3trU9K3NzcUFBQgPz8fP1xQUFBcHd31y/7+vqipKSk2vNMmzYNV69e1bfi4uJavS4iIrIdAhJ0MpqQMV+kNtn8hNPQ0FBIklTlvuTg4GAAgIuLC4Bbtxf5+vre83YhT09P/deOjo4G2yRJ0j/17V5UKpVJD3MhIiKyVTaffHh7e6Nnz55YtmwZXn311WrnfcTExODixYtwcHBAUFCQZYMkIiK6B7lDJxx2saL3338fN2/eRGxsLP7zn/8gJycHubm5+Ne//oWzZ8/C3t4e8fHx6NChA/r06YM9e/agsLAQ33zzDd544w1kZmZa+xKIiKgeUtJbbc3J5isfABASEoJTp05h3rx5mDZtGn7++WeoVCq0bNkSkyZNwujRoyFJEnbu3Ik33ngDI0aMwO+//w4fHx907doVTZs2tfYlEBER2QxJPOhpImRWpaWl8PDwwOOh4+FgX7fmglQ+5GHtEEwi3ah+To7SSbq6+eMpfZP94J0UyMHXx9ohmESnKbN2CCbTRoZYOwSj3bxZjgNH3sTVq1dlvy+lOrd/V4w//AxUbo4PPqAaFZobeLfTtlqN1RT1ovJBRERUF8kdOlHqsEu9mPNBREREysHKBxERkULpYAedjDqBnGNrE5MPIiIihdIKCVoZQydyjq1NykyJiIiIyGax8kFERKRQtjrhlMkHERGRQgmZb6YVCn3CKZMPIiIihdJCglbGy+HkHFublJkSERERkc1i5YOIiEihdELevA2lPiSZyQcREZFC6WTO+ZBzbG1SZlRERERks1j5ICIiUigdJOhkTBqVc2xtYvJBRESkUHzCKREREZEZsPJhJX881hj2Ts7WDsMoFQ2VmUE/yA03a0dgOvsKa0dgGq/G7awdgkmEXd38jNuX66wdgsmuhDlaOwSjaSvsgCOWOZetTjhl8kFERKRQOsh8vLpC53woMyUiIiIim8XKBxERkUIJmXe7CIVWPph8EBERKRTfaktEREQWZasTTpUZFREREdksVj6IiIgUisMuREREZFG2+nh1DrsQERERACAjIwOJiYnw8/ODJEnYunWrwfbhw4dDkiSD1rt3b6PPw8oHERGRQll62KWsrAxRUVEYOXIk+vXrd899evfujXXr1umXVSqV0XEx+SAiIlIoSycfCQkJSEhIuO8+KpUKPj4+JscEcNiFiIjI5pWWlhq0igrTXxyVnp6OJk2aoHnz5nj55Zdx6dIlo/tg8kFERKRQtysfchoA+Pv7w8PDQ99SUlJMiqd37974+OOPsW/fPsyfPx8HDhxAQkICtFqtUf1w2IWIiEihzDXsUlxcDLVarV9vyjwNABg0aJD+69atWyMyMhIhISFIT0/H448/XuN+WPkgIiKycWq12qCZmnzcLTg4GI0aNUJeXp5Rx7HyQUREpFAC8p7VIcwXyj39/PPPuHTpEnx9fY06jskHERGRQln6bheNRmNQxSgoKEBWVha8vLzg5eWF5ORk9O/fHz4+PsjPz8eUKVMQGhqKXr16GXUeJh9EREQKZenkIzMzE927d9cvT5gwAQAwbNgwrFixAqdPn8ZHH32EK1euwM/PD0888QTmzp1r9DAOkw8AcXFxaNOmDd59911rh0JERGQ1cXFxEKL6wZrdu3eb5TxMPgCkpaXB0dHR2mEQEREZ4IvlbJiXl5e1QyAiIqrCVpMP3mqLW2Wm8ePHA8A9X6Tj6emJ9evXAwAqKyvxyiuvwNfXF87OzggMDDT5YS1ERET1ESsfRlq6dCm2bduGzz77DAEBASguLkZxcXG1+1dUVBg8xra0tNQSYRIRkQ0QQoKQUb2Qc2xtYvJhpKKiIoSFhaFz586QJAmBgYH33T8lJQXJyckWio6IiGyJDpKs53zIObY2cdjFSMOHD0dWVhaaN2+OsWPHYs+ePffdf9q0abh69aq+3a9KQkREVB8w+biLJElVbjO6ceOG/uuYmBgUFBRg7ty5+Ouvv5CUlIQBAwZU259KparyWFsiIqKaMNeL5ZSGwy53ady4MS5cuKBfPnfuHK5fv26wj1qtxsCBAzFw4EAMGDAAvXv3xuXLl3nXDBERmRXnfNQTPXr0wLJly9ChQwdotVpMnTrV4Bkgixcvhq+vL6Kjo2FnZ4dNmzbBx8cHnp6e1guaiIioDmHycZdFixZhxIgR6NKlC/z8/LBkyRKcOHFCv93d3R2pqak4d+4c7O3t0bZtW+zcuRN2dhzBIiIi87LV53ww+QCQnp6u/9rPz6/K42OvXLmi/3rUqFEYNWqUhSIjIqL6jMMuREREZFFCZuVDqckHxwqIiIjIolj5ICIiUigB4D4vma3R8UrE5IOIiEihdJAg8QmnRERERPKw8kFERKRQvNuFiIiILEonJEg2+JwPDrsQERGRRbHyQUREpFBCyLzbRaG3uzD5ICIiUihbnfPBYRciIiKyKFY+iIiIFMpWKx9MPoiIiBTKVu92YfJBRESkULY64ZRzPoiIiMiiWPkgIiJSqFuVDzlzPswYjBkx+bCSa4GAvbO1ozBOpZfW2iGYxPmCvbVDMNmkEZ9bOwSTJKf3sXYIJvE6WTeLwS6XrB2B6R7922lrh2C0Sk0lflhpmXPZ6oTTuvmTRkRERHUWKx9EREQKJf7X5ByvREw+iIiIFIrDLkRERERmwMoHERGRUtnouAsrH0REREr1v2EXUxuMHHbJyMhAYmIi/Pz8IEkStm7dWu2+//znPyFJEt59912jL4vJBxERkULdfsKpnGaMsrIyREVFYfny5ffdb8uWLThy5Aj8/PxMui4OuxAREREAICEhAQkJCffd55dffsGrr76K3bt346mnnjLpPEw+iIiIFMpcd7uUlpYarFepVFCpVEb3p9PpMGTIEEyePBmtWrUyOS4OuxARESnV7XkbchoAf39/eHh46FtKSopJ4cyfPx8ODg4YO3asrMti5YOIiMjGFRcXQ61W65dNqXqcOHECS5YswcmTJyFJ8p4fwsoHERGRQplrwqlarTZopiQfBw8eRElJCQICAuDg4AAHBwecP38eEydORFBQkFF9sfJBRESkVAp6zseQIUMQHx9vsK5Xr14YMmQIRowYYVRfTD6IiIgIAKDRaJCXl6dfLigoQFZWFry8vBAQEABvb2+D/R0dHeHj44PmzZsbdR4mH0RERApl6Xe7ZGZmonv37vrlCRMmAACGDRuG9evXmxzH3Zh8EBERKZkFH5EeFxcHYcSTyQoLC006DyecEhERkUUx+aihuLg4jB8/3tphEBFRPSLnvS5yh2xqE4ddaigtLQ2Ojo7WDoOIiOoTBd3tYk5MPmrIy8vL2iEQEVG9I/2vyTleeTjsUkN3Dru8//77CAsLg7OzM5o2bYoBAwZYNzgiIqI6hJUPI2VmZmLs2LH45JNP0LFjR1y+fBkHDx6sdv+KigpUVFTol+9+uQ8REVG1OOxCAFBUVARXV1c8/fTTcHd3R2BgIKKjo6vdPyUlBcnJyRaMkIiIbIaNJh8cdjFSz549ERgYiODgYAwZMgSffvoprl+/Xu3+06ZNw9WrV/WtuLjYgtESEREpD5MPI7m7u+PkyZPYsGEDfH19MXPmTERFReHKlSv33F+lUlV5oQ8REVGNCEl+UyAmHyZwcHBAfHw8UlNTcfr0aRQWFuLrr7+2dlhERGRjzPVWW6XhnA8jbd++HT/99BO6du2Khg0bYufOndDpdEa/VIeIiKi+YvJhJE9PT6SlpWH27NkoLy9HWFgYNmzYgFatWlk7NCIisjU2OuGUyUcNpaen3/NrIiKiWiN33gbnfBARERGx8kFERKRYkrjV5ByvREw+iIiIlIpzPoiIiMiiOOeDiIiISD5WPoiIiJSKwy5ERERkUTaafHDYhYiIiCyKlQ8iIiKlstHKB5MPIiIipeLdLkRERETysfJBRESkUHzCKREREVmWjc754LALERERWRSTDyIiIrIoDrsQEREplASZcz7MFol5MfmwkqBtV+FgX27tMIxyOcrT2iGY5HpTa0dguuHqEmuHYJLMR7OtHYJJdt5oY+0QTCKdrrtF7JAGv1s7BKOV625Y7mS81ZaIiIhIPlY+iIiIlMpG73Zh8kFERKRUNpp8cNiFiIiIAAAZGRlITEyEn58fJEnC1q1bDbbPnj0bLVq0gKurKxo2bIj4+HgcPXrU6PMw+SAiIlKo2084ldOMUVZWhqioKCxfvvye28PDw7Fs2TKcOXMGhw4dQlBQEJ544gn8/rtxE4c57EJERKRUFh52SUhIQEJCQrXb//a3vxksL168GGvXrsXp06fx+OOP1/g8TD6IiIhsXGlpqcGySqWCSqWS1WdlZSVWr14NDw8PREVFGXUsh12IiIiUSpihAfD394eHh4e+paSkmBzS9u3b4ebmBmdnZ7zzzjvYu3cvGjVqZFQfrHwQEREplLnealtcXAy1Wq1fL6fq0b17d2RlZeGPP/7AmjVrkJSUhKNHj6JJkyY17oOVDyIiIhunVqsNmpzkw9XVFaGhoXjsscewdu1aODg4YO3atUb1wcoHERGRUtWBx6vrdDpUVFQYdQyTDyIiIqWy8N0uGo0GeXl5+uWCggJkZWXBy8sL3t7eeOutt/DMM8/A19cXf/zxB5YvX45ffvkFzz33nFHnYfJBRESkUOaa81FTmZmZ6N69u355woQJAIBhw4Zh5cqVOHv2LD766CP88ccf8Pb2Rtu2bXHw4EG0atXKqPMw+SAiIiIAQFxcHISoPmNJS0szy3mYfBARESmVjb7bhckHERGRUskcdlFq8sFbbYmIiMiiWPkgIiJSKg67EBERkUXZaPLBYRciIiKyqHqVfFRUVGDs2LFo0qQJnJ2d0blzZxw/fhwAkJ6eDkmSsG/fPsTGxqJBgwbo2LEjcnNzDfr44osvEBMTA2dnZwQHByM5ORk3b960xuUQEZGNu/2cDzlNiepV8jFlyhRs3rwZH330EU6ePInQ0FD06tULly9f1u/zxhtvYNGiRcjMzISDgwNGjhyp33bw4EEMHToU48aNww8//IBVq1Zh/fr1eOutt6xxOURERHVSvUk+ysrKsGLFCixYsAAJCQlo2bIl1qxZAxcXF4MX4rz11lvo1q0bWrZsiddffx3ffPMNysvLAQDJycl4/fXXMWzYMAQHB6Nnz56YO3cuVq1aVe15KyoqUFpaatCIiIjqs3qTfOTn5+PGjRvo1KmTfp2joyPatWuHnJwc/brIyEj9176+vgCAkpISAEB2djbmzJkDNzc3fRs1ahQuXLiA69ev3/O8KSkp8PDw0Dd/f//auDwiIrJFwgxNgXi3y10cHR31X0vSrbcB6nQ6ALdeuJOcnIx+/fpVOc7Z2fme/U2bNk3/bHwAKC0tZQJCREQ1Yul3u1hKvUk+QkJC4OTkhMOHDyMwMBAAcOPGDRw/fhzjx4+vUR8xMTHIzc1FaGhojc+rUqmgUqlMCZmIiEix1Qs56k3y4erqipdffhmTJ0+Gl5cXAgICkJqaiuvXr+OFF15Adnb2A/uYOXMmnn76aQQEBGDAgAGws7NDdnY2vvvuO7z55psWuAoiIqK6r94kHwDw9ttvQ6fTYciQIbh27RpiY2Oxe/duNGzYsEbH9+rVC9u3b8ecOXMwf/58ODo6okWLFnjxxRdrOXIiIqqXbPQhY/Uq+XB2dsbSpUuxdOnSKtvu9RrhNm3aVFnXq1cv9OrVq1bjJCIiAmx3zke9uduFiIiIlKFeVT6IiIjqFA67EBERkSVx2IWIiIjIDFj5ICIiUioOuxAREZFF2WjywWEXIiIisihWPoiIiBTKViecMvkgIiJSKhsddmHyQUREpFQ2mnxwzgcRERFZFCsfRERECsU5H0RERGRZHHYhIiIiko+VDyIiIoXisAsRERFZlo0OuzD5sBat3E+U5al/+svaIZikYY7W2iGYLDR6uLVDMIl9vou1QzCJSitZOwSTSNq69X/Jndae7mjtEIymu14OYI+1w6jTmHwQEREplY1WPjjhlIiISKEkMzRjZGRkIDExEX5+fpAkCVu3btVvu3HjBqZOnYrWrVvD1dUVfn5+GDp0KH799Vejr4vJBxEREQEAysrKEBUVheXLl1fZdv36dZw8eRIzZszAyZMnkZaWhtzcXDzzzDNGn4fDLkREREpl4WGXhIQEJCQk3HObh4cH9u7da7Bu2bJlaNeuHYqKihAQEFDj8zD5ICIiUihz3WpbWlpqsF6lUkGlUsmI7JarV69CkiR4enoadRyHXYiIiJRKmKEB8Pf3h4eHh76lpKTIDq28vBxTp07F4MGDoVarjTqWlQ8iIiIbV1xcbJAgyK163LhxA0lJSRBCYMWKFUYfz+SDiIhIycxwu6xarTa6OlGd24nH+fPn8fXXX5vUL5MPIiIihVLa49VvJx7nzp3D/v374e3tbVI/TD6IiIgIAKDRaJCXl6dfLigoQFZWFry8vODr64sBAwbg5MmT2L59O7RaLS5evAgA8PLygpOTU43Pw+SDiIhIqSx8q21mZia6d++uX54wYQIAYNiwYZg9eza2bdsGAGjTpo3Bcfv370dcXFyNz8Pkg4iISKEsPewSFxcHIao/6H7bjMFbbYmIiMiiWPkgIiJSKht9sRyTDyIiIoVS2t0u5sJhFyIiIrIoVj6IiIiUisMuREREZFFMPoiIiMiSOOeDqpAkCVu3brV2GERERHWKzVU+tFotJEmCnR3zKiIiquNsdNjF6r+h4+Li8Morr+CVV16Bh4cHGjVqhBkzZuifolZRUYFJkybhoYcegqurK9q3b4/09HT98evXr4enpye2bduGli1bQqVSoaioCBUVFZg6dSr8/f2hUqkQGhqKtWvX6o/77rvvkJCQADc3NzRt2hRDhgzBH3/8YRDX2LFjMWXKFHh5ecHHxwezZ8/Wbw8KCgIA9O3bF5Ik6ZeJiIjMRRJCdlMiqycfAPDRRx/BwcEBx44dw5IlS7B48WJ88MEHAIBXXnkF3377LTZu3IjTp0/jueeeQ+/evXHu3Dn98devX8f8+fPxwQcf4Pvvv0eTJk0wdOhQbNiwAUuXLkVOTg5WrVoFNzc3AMCVK1fQo0cPREdHIzMzE7t27cJvv/2GpKSkKnG5urri6NGjSE1NxZw5c7B3714AwPHjxwEA69atw4ULF/TLd6uoqEBpaalBIyIiqs8UMezi7++Pd955B5IkoXnz5jhz5gzeeecd9OrVC+vWrUNRURH8/PwAAJMmTcKuXbuwbt06zJs3D8CtV/y+//77iIqKAgD8+OOP+Oyzz7B3717Ex8cDAIKDg/XnW7ZsGaKjo/XHA8CHH34If39//PjjjwgPDwcAREZGYtasWQCAsLAwLFu2DPv27UPPnj3RuHFjAICnpyd8fHyqvbaUlBQkJyeb61tFRET1CYddas9jjz0GSZL0yx06dMC5c+dw5swZaLVahIeHw83NTd8OHDiA/Px8/f5OTk6IjIzUL2dlZcHe3h7dunW75/mys7Oxf/9+gz5btGgBAAb93tknAPj6+qKkpMSoa5s2bRquXr2qb8XFxUYdT0RE9dftu13kNCVSROWjOhqNBvb29jhx4gTs7e0Ntt0eQgEAFxcXg+TFxcXlgf0mJiZi/vz5Vbb5+vrqv3Z0dDTYJkkSdDqdUdegUqmgUqmMOoaIiMiWKSL5OHr0qMHykSNHEBYWhujoaGi1WpSUlKBLly417q9169bQ6XQ4cOCAftjlTjExMdi8eTOCgoLg4GD6t8DR0RFardbk44mIiO6Lwy61p6ioCBMmTEBubi42bNiA9957D+PGjUN4eDief/55DB06FGlpaSgoKMCxY8eQkpKCHTt2VNtfUFAQhg0bhpEjR2Lr1q0oKChAeno6PvvsMwDAmDFjcPnyZQwePBjHjx9Hfn4+du/ejREjRhiVTAQFBWHfvn24ePEi/vzzT9nfByIiojvZ6rCLIpKPoUOH4q+//kK7du0wZswYjBs3Di+99BKAW3eTDB06FBMnTkTz5s3Rp08fHD9+HAEBAfftc8WKFRgwYABGjx6NFi1aYNSoUSgrKwMA+Pn54fDhw9BqtXjiiSfQunVrjB8/Hp6enkY9H2TRokXYu3cv/P39ER0dbfo3gIiIqB6RhLDuTcBxcXFo06YN3n33XWuGYTGlpaXw8PBAj5aT4WBft+aC3PR0tnYIJrGvqLtDY3mv2T94JwWyz7//vCulstNKD95JgdyKFPrnbQ1c7lFu7RCMprtejqJRc3H16lWo1epaOcft3xUxg96CvZPp//dqK8txcuMbtRqrKRQx54OIiIiqstV3uzD5ICIiUiobnXBq9eTjzkelExERke2zevJBRERE1VPq0IkcTD6IiIiUSohbTc7xCqSIW22JiIio/mDlg4iISKF4twsRERFZlo3e7cJhFyIiIrIoVj6IiIgUStLdanKOVyImH0RERErFYRciIiIi+Vj5ICIiUije7UJERESWZaMPGWPyQUREpFCsfJBZ2V26Ajs7J2uHYRTH8gbWDsE0VzXWjsBkDb8KtXYIJnH/+Ya1QzBJecO6+V+ik0Zr7RBMVpbjYu0QjKatkKwdQq3JyMjAggULcOLECVy4cAFbtmxBnz599NvT0tKwcuVKnDhxApcvX8apU6fQpk0bo8/DCadERERKJczQjFBWVoaoqCgsX7682u2dO3fG/PnzTbiY/1M303wiIqJ6wNLDLgkJCUhISKh2+5AhQwAAhYWFpgcFJh9EREQ2r7S01GBZpVJBpVJZKRoOuxARESnX7btd5DQA/v7+8PDw0LeUlBSrXhYrH0RERAplrmGX4uJiqNVq/XprVj0AJh9EREQ2T61WGyQf1sbkg4iISKls9N0uTD6IiIgUytJ3u2g0GuTl5emXCwoKkJWVBS8vLwQEBODy5csoKirCr7/+CgDIzc0FAPj4+MDHx6fG5+GEUyIiIgIAZGZmIjo6GtHR0QCACRMmIDo6GjNnzgQAbNu2DdHR0XjqqacAAIMGDUJ0dDRWrlxp1HlY+SAiIlIqnbjV5BxvhLi4OIj7vA9m+PDhGD58uOnx/A+TDyIiIqXinA8iIiKyJAky53yYLRLz4pwPIiIisihWPoiIiJTqjqeUmny8AjH5ICIiUihL32prKRx2ISIiIoti5YOIiEipeLcLERERWZIkBCQZ8zbkHFubjBp2iYuLw/jx42spFEOFhYWQJAlZWVm10v/69evh6elZK30TERFR9YyqfKSlpcHR0bG2YjHg7++PCxcuoFGjRrL7CgoKwvjx4w0Sp4EDB+LJJ5+U3TcREVGt0f2vyTlegYxKPry8vGorjirs7e3v+5IaIQS0Wi0cHEwbOXJxcYGLi4up4REREdU6DrvAcNglKCgI8+bNw8iRI+Hu7o6AgACsXr1av29lZSVeeeUV+Pr6wtnZGYGBgUhJSdFvlyQJK1asQEJCAlxcXBAcHIzPP/9cv/3uYZf09HRIkoQvv/wSjz76KFQqFQ4dOoT8/Hw8++yzaNq0Kdzc3NC2bVt89dVXBjGfP38er732GiRJgiTdet7bvYZdVqxYgZCQEDg5OaF58+b45JNPDLZLkoQPPvgAffv2RYMGDRAWFoZt27YZ8y0kIiKq92Tdarto0SLExsbi1KlTGD16NF5++WX963WXLl2Kbdu24bPPPkNubi4+/fRTBAUFGRw/Y8YM9O/fH9nZ2Xj++ecxaNAg5OTk3Pecr7/+Ot5++23k5OQgMjISGo0GTz75JPbt24dTp06hd+/eSExMRFFREYBbQ0XNmjXDnDlzcOHCBVy4cOGe/W7ZsgXjxo3DxIkT8d133+Ef//gHRowYgf379xvsl5ycjKSkJJw+fRpPPvkknn/+eVy+fLnaeCsqKlBaWmrQiIiIakSYoSmQrOTjySefxOjRoxEaGoqpU6eiUaNG+l/WRUVFCAsLQ+fOnREYGIjOnTtj8ODBBsc/99xzePHFFxEeHo65c+ciNjYW77333n3POWfOHPTs2RMhISHw8vJCVFQU/vGPf+CRRx5BWFgY5s6di5CQEH1FwsvLC/b29nB3d4ePj0+1QzkLFy7E8OHDMXr0aISHh2PChAno168fFi5caLDf8OHDMXjwYISGhmLevHnQaDQ4duxYtfGmpKTAw8ND3/z9/R/4fSUiIgLwf084ldMUSFbyERkZqf9akiT4+PigpKQEwK1f0llZWWjevDnGjh2LPXv2VDm+Q4cOVZYfVPmIjY01WNZoNJg0aRIiIiLg6ekJNzc35OTk6CsfNZWTk4NOnToZrOvUqVOVeO68ZldXV6jVav0138u0adNw9epVfSsuLjYqLiIiqr9uP+FUTlMiWc/5uPvOF0mSoNPdmlobExODgoICfPnll/jqq6+QlJSE+Ph4g3kdpnB1dTVYnjRpEvbu3YuFCxciNDQULi4uGDBgACorK2Wdpzr3u+Z7UalUUKlUtRILERFRXVSrj1dXq9UYOHAg1qxZg//85z/YvHmzwfyII0eOGOx/5MgRREREGHWOw4cPY/jw4ejbty9at24NHx8fFBYWGuzj5OQErVZ7334iIiJw+PDhKn23bNnSqHiIiIjMxkaHXWrtCaeLFy+Gr68voqOjYWdnh02bNsHHx8fgDpNNmzYhNjYWnTt3xqeffopjx45h7dq1Rp0nLCwMaWlpSExMhCRJmDFjRpVKRFBQEDIyMjBo0CCoVKp7Pjtk8uTJSEpKQnR0NOLj4/Hf//4XaWlpBnfOEBERWZKku9XkHK9EtVb5cHd3R2pqKmJjY9G2bVsUFhZi586dsLP7v1MmJydj48aNiIyMxMcff4wNGzYYXWlYvHgxGjZsiI4dOyIxMRG9evVCTEyMwT5z5sxBYWEhQkJC0Lhx43v206dPHyxZsgQLFy5Eq1atsGrVKqxbtw5xcXFGXzsRERFVTxLCOjUZSZKwZcsW9OnTxxqnt5rS0lJ4eHggvukoONg5WTscowi3BtYOwTRXNdaOwGS/J4ZaOwSTuP98w9ohmKS8Yd183ZWT5v7DykpW0sYyT802J21FOfIW/D9cvXoVarW6Vs5x+3dFXLs34ODgbHI/N2+WI/3YW7Uaqynq5k8aERFRfWCjb7Wt1QmnRERERHezWuXDSqM9REREdYatvtuFwy5ERERKJfd2WYUmHxx2ISIiIoti5YOIiEipBAA5z+pQZuGDyQcREZFScc4HERERWZaAzDkfZovErDjng4iIiCyKlQ8iIiKlstG7XZh8EBERKZUOgCTzeAXisAsRERFZFCsfRERECmWrd7uw8kFERKRUt+d8yGlGyMjIQGJiIvz8/CBJErZu3XpXOAIzZ86Er68vXFxcEB8fj3Pnzhl9WUw+iIiICABQVlaGqKgoLF++/J7bU1NTsXTpUqxcuRJHjx6Fq6srevXqhfLycqPOw2EXIiIipbLw3S4JCQlISEiopiuBd999F9OnT8ezzz4LAPj444/RtGlTbN26FYMGDarxeZh8WEnByIdh7+xs7TDqBbciZY551oTvsAJrh2CSH44HWTsEkzhq6mYx2K6ybsYNAAOT0q0dgtEqNDcwf4GFTmam5KO0tNRgtUqlgkqlMqqrgoICXLx4EfHx8fp1Hh4eaN++Pb799lujko+6+4klIiKiGvH394eHh4e+paSkGN3HxYsXAQBNmzY1WN+0aVP9tppi5YOIiEipzPScj+LiYqjVav1qY6se5sbKBxERkULdvtVWTgMAtVpt0ExJPnx8fAAAv/32m8H63377Tb+tpph8EBERKZWFb7W9n4cffhg+Pj7Yt2+ffl1paSmOHj2KDh06GNUXh12IiIgIAKDRaJCXl6dfLigoQFZWFry8vBAQEIDx48fjzTffRFhYGB5++GHMmDEDfn5+6NOnj1HnYfJBRESkVDoBSDKqFzrjjs3MzET37t31yxMmTAAADBs2DOvXr8eUKVNQVlaGl156CVeuXEHnzp2xa9cuOBt59yaTDyIiIqWy8HM+4uLiIO5zjCRJmDNnDubMmWN6TOCcDyIiIrIwVj6IiIgUS+6kUWU+ZJHJBxERkVJZeNjFUjjsQkRERBbFygcREZFS6QRkDZ0YebeLpTD5ICIiUiqhu9XkHK9AHHYhIiIii2Llg4iISKlsdMIpkw8iIiKl4pwPIiIisigbrXxwzodM69evh6enp7XDICIiqjNY+SAiIlIqAZmVD7NFYlZMPoiIiJSKwy51365du9C5c2d4enrC29sbTz/9NPLz8wEAhYWFkCQJaWlp6N69Oxo0aICoqCh8++23Bn2sX78eAQEBaNCgAfr27YtLly5Z41KIiIjqrHqVfJSVlWHChAnIzMzEvn37YGdnh759+0Kn+7+HsLzxxhuYNGkSsrKyEB4ejsGDB+PmzZsAgKNHj+KFF17AK6+8gqysLHTv3h1vvvnmfc9ZUVGB0tJSg0ZERFQjOp38pkD1atilf//+BssffvghGjdujB9++AFubm4AgEmTJuGpp54CACQnJ6NVq1bIy8tDixYtsGTJEvTu3RtTpkwBAISHh+Obb77Brl27qj1nSkoKkpOTa+mKiIjIpnHYpe47d+4cBg8ejODgYKjVagQFBQEAioqK9PtERkbqv/b19QUAlJSUAABycnLQvn17gz47dOhw33NOmzYNV69e1bfi4mJzXAoREVGdVa8qH4mJiQgMDMSaNWvg5+cHnU6HRx55BJWVlfp9HB0d9V9LkgQABsMyxlKpVFCpVKYHTURE9ZeNVj7qTfJx6dIl5ObmYs2aNejSpQsA4NChQ0b1ERERgaNHjxqsO3LkiNliJCIiMsAnnNZtDRs2hLe3N1avXg1fX18UFRXh9ddfN6qPsWPHolOnTli4cCGeffZZ7N69+77zPYiIiKiqejPnw87ODhs3bsSJEyfwyCOP4LXXXsOCBQuM6uOxxx7DmjVrsGTJEkRFRWHPnj2YPn16LUVMRET1nRA62U2J6k3lAwDi4+Pxww8/GKwTd4yHibvGxjw9PausGzlyJEaOHGmwbuLEiWaOlIiICLfmbMgZOuGcDyIiIjKKkDnnQ6HJR70ZdiEiIiJlYOWDiIhIqXQ6QJIxb4NzPoiIiMgoHHYhIiIiko+VDyIiIoUSOh2EjGEX3mpLRERExuGwCxEREZF8rHwQEREplU4Aku1VPph8EBERKZUQAOTcaqvM5IPDLkRERGRRTD6IiIgUSuiE7Gasa9euYfz48QgMDISLiws6duyI48ePm/W6mHwQEREpldDJb0Z68cUXsXfvXnzyySc4c+YMnnjiCcTHx+OXX34x22Ux+SAiIlIoS1c+/vrrL2zevBmpqano2rUrQkNDMXv2bISGhmLFihVmuy5OOCUiIrJxpaWlBssqlQoqlarKfjdv3oRWq4Wzs7PBehcXFxw6dMhs8TD5sDDxv5nHuopyK0dSf2grlTnbuyZulFVaOwST6Mrr5udbW143i8HihrUjMF2Fpu4FX1F2K2ZhgTtJbooKWS+Hu4lbsfr7+xusnzVrFmbPnl1lf3d3d3To0AFz585FREQEmjZtig0bNuDbb79FaGioyXHcTRKW+O6R3s8//1zlQ0BERHVPcXExmjVrVit9l5eX4+GHH8bFixdl9+Xj44Ps7GyDakZ1lQ8AyM/Px8iRI5GRkQF7e3vExMQgPDwcJ06cQE5Ojux4ACYfFqfT6fDrr7/C3d0dkiSZte/S0lL4+/ujuLgYarXarH3XtroaO+O2LMZteXU19tqMWwiBa9euwc/PD3Z2tVctKy8vR2Wl/Oqnk5NTlWGUmigrK0NpaSl8fX0xcOBAaDQa7NixQ3Y8AIddLM7Ozq7WMuXb1Gp1nfpP4k51NXbGbVmM2/Lqauy1FbeHh4fZ+7ybs7OzSUmDubi6usLV1RV//vkndu/ejdTUVLP1zeSDiIiI9Hbv3g0hBJo3b468vDxMnjwZLVq0wIgRI8x2jro5u4qIiIhqxdWrVzFmzBi0aNECQ4cORefOnbF79244Ojqa7RysfNgQlUqFWbNmVTuJSMnqauyM27IYt+XV1djratxKkJSUhKSkpFo9ByecEhERkUVx2IWIiIgsiskHERERWRSTDyIiIrIoJh9E9xEXF4fx48dbOwyzsKVrsXWW/LcqLCyEJEnIysqqlf7Xr18PT0/PWunbHJQen63ihFOqNRcvXkRKSgp27NiBn3/+GR4eHggNDcXf//53DBs2DA0aNLB2iA90+fJlODo6wt3d3dqhyGZL12LrjPm3kiQJW7ZsQZ8+fUw6l1arxe+//45GjRrBwUHeDZBBQUEYP368QeL0119/4dq1a2jSpImsvmvL+vXrMX78eFy5csXaodQrvNWWasVPP/2ETp06wdPTE/PmzUPr1q2hUqlw5swZrF69Gg899BCeeeYZa4f5QF5eXtYOwWxs6VqUTqvVQpIkkx+9bcl/K3t7e/j4+FS7XQgBrVZrcmLi4uICFxcXU8MjWyWIakGvXr1Es2bNhEajued2nU4nhBDizz//FC+88IJo1KiRcHd3F927dxdZWVn6/WbNmiWioqLExx9/LAIDA4VarRYDBw4UpaWlFrmObt26iXHjxgkhhAAgtmzZYrDdw8NDrFu3TgghREVFhRgzZozw8fERKpVKBAQEiHnz5lkkzpq481qWL18uQkNDhUqlEk2aNBH9+/e3amzl5eXi1VdfFY0bNxYqlUp06tRJHDt2TAghxP79+wUA8dVXX4lHH31UuLi4iA4dOoizZ88a9LF161YRHR0tVCqVePjhh8Xs2bPFjRs3anT+bt26iTFjxogxY8YItVotvL29xfTp0/Wf0/LycjFx4kTh5+cnGjRoINq1ayf279+vP37dunXCw8NDfPHFFyIiIkLY29uLgoICUV5eLqZMmSKaNWsmnJycREhIiPjggw/0x505c0b07t1buLq6iiZNmoi///3v4vfff9f/W3Xr1k24u7uLLl26CCcnJyFJklCr1WLVqlVCCCECAwMFAIN29+cOgHj//fdF7969hbOzs3j44YfFpk2b9NsLCgoEAHHq1CmD7/fOnTtFTEyMcHR0FPv37xd5eXnimWeeEU2aNBGurq4iNjZW7N271+B7eHcsd35v7vT++++L4OBg4ejoKMLDw8XHH39ssB2AWLNmjejTp49wcXERoaGh4osvvhBffvml6NSpk/Dw8BBeXl7iqaeeEnl5eQbXsXnzZhEXFydcXFxEZGSk+Oabbwz6XrdunfD39xcuLi6iT58+YuHChVXio9rH5IPM7o8//hCSJImUlJQH7hsfHy8SExPF8ePHxY8//igmTpwovL29xaVLl4QQt5IPNzc30a9fP3HmzBmRkZEhfHx8xP/7f/+vti9DCGFc8rFgwQLh7+8vMjIyRGFhoTh48KD497//bZE4a+L2tRw/flzY29uLf//736KwsFCcPHlSLFmyxKqxjR07Vvj5+YmdO3eK77//XgwbNkw0bNhQXLp0Sf/LsH379iI9PV18//33okuXLqJjx4764zMyMoRarRbr168X+fn5Ys+ePSIoKEjMnj27Rufv1q2bcHNzE+PGjRNnz54V//rXv0SDBg3E6tWrhRBCvPjii6Jjx44iIyND5OXliQULFgiVSiV+/PFHIcStX2iOjo6iY8eO4vDhw+Ls2bOirKxMJCUlCX9/f5GWliby8/PFV199JTZu3CiEuJV4N27cWEybNk3k5OSIkydPip49e4ru3bsbJB+SJAkXFxcxa9YskZqaKgAIOzs7cfbsWVFSUiIACC8vL7FlyxZx4sSJKp87AMLb21usWbNG5ObmiunTpwt7e3vxww8/CCGqTz4iIyPFnj17RF5enrh06ZLIysoSK1euFGfOnBE//vijmD59unB2dhbnz58XQghx6dIl0axZMzFnzhxx4cIFceHCBf335s5f7mlpacLR0VEsX75c5ObmikWLFgl7e3vx9ddfG8TcrFkz8e9//1ucO3dOjB07Vri5uYl169aJzZs3i3PnzolTp06JxMRE0bp1a6HVavXX0aJFC7F9+3aRm5srBgwYIAIDA/VJ6JEjR4SdnZ2YP3++yM3NFUuWLBGenp5MPqyAyQeZ3ZEjRwQAkZaWZrDe29tbuLq6CldXVzFlyhRx8OBBoVarRXl5ucF+ISEh+r/sZs2aJRo0aGBQ6Zg8ebJo37597V+IMC75ePXVV0WPHj30fy0rze1r2bx5s1Cr1RarHj2IRqMRjo6O4tNPP9Wvq6ysFH5+fiI1NdWg8nHbjh07BADx119/CSGEePzxx6tUmT755BPh6+tboxi6desmIiIiDP7tpk6dKiIiIsT58+eFvb29+OWXXwyOefzxx8W0adOEELd+wQIwqNrl5uYKAAbVgTvNnTtXPPHEEwbriouLBQDRrl07ffKhUqnE3//+d/0+sbGxokGDBmLFihVCiFufy9atW1f7uQMg/vnPfxqsa9++vXj55ZeFENUnH1u3bq32+3Vbq1atxHvvvadfDgwMFO+8847BPncnHx07dhSjRo0y2Oe5554TTz75pEHM06dP1y9rNBoBQHz55ZcGx/3+++8CgDhz5oz+Ou6sLH3//fcCgMjJyRFCCDF48GCD8wghxMCBA5l8WAHvdiGLOXbsGLKystCqVStUVFQgOzsbGo0G3t7ecHNz07eCggLk5+frjwsKCjKYeOfr64uSkhJrXMJ9DR8+HFlZWWjevDnGjh2LPXv2WDuke+rZsycCAwMRHByMIUOG4NNPP8X169etFk9+fj5u3LiBTp066dc5OjqiXbt2yMnJ0a+LjIzUf+3r6wsA+s9BdnY25syZY/A5GjVqFC5cuFDja3vssccgSZJ+uUOHDjh37hzOnDkDrVaL8PBwg/4PHDhg8Dl1cnIyiDErKwv29vbo1q3bPc+XnZ2N/fv3G/TZokULALcmaVbXr5+fH1QqlcHPQEFBwX0/dx06dKiyfOf39l5iY2MNljUaDSZNmoSIiAh4enrCzc0NOTk5KCoqum8/d8vJyTH4twaATp06VYnnzmt2dXWFWq3G6dOnMXjwYAQHB0OtViMoKAgADGK43+ckJycH7du3NzjP3d8bsgxOOCWzCw0NhSRJyM3NNVgfHBwMAPrJZxqNBr6+vkhPT6/Sx523vt39MiNJkqDT6cwbdA1IkgRx181hN27c0H8dExODgoICfPnll/jqq6+QlJSE+Ph4fP7555YO9b7c3d1x8uRJpKenY8+ePZg5cyZmz56N48ePK/qWwzs/B7eThNufA41Gg+TkZPTr16/KcXJfSa7RaGBvb48TJ07A3t7eYJubm5v+axcXF4Pk5UGTLDUaDRITEzF//vwq2+5+e+j9rh0AVq1aBXt7e7N+7lxdXQ2WJ02ahL1792LhwoUIDQ2Fi4sLBgwYgMrKSlnnqc69fu7feecdREZGYs2aNfDz84NOp8MjjzxiEMODvlekDEw+yOy8vb3Rs2dPLFu2DK+++mqV/8Rui4mJwcWLF+Hg4KD/C0bJGjdujAsXLuiXz507V+WvarVajYEDB2LgwIEYMGAAevfujcuXLyvuThMHBwfEx8cjPj4es2bNgqenJ77++ut7/vKubSEhIXBycsLhw4cRGBgI4FZSd/z48Ro/6yImJga5ubkIDQ01OY6jR48aLB85cgRhYWGIjo6GVqtFSUkJunTpUuP+WrduDZ1OhwMHDiA+Pv6eMW/evBlBQUFV7iS5O8m5H0dHR6hUKvTv37/az92RI0cwdOhQg2uLjo6u8TkA4PDhwxg+fDj69u0L4FbyVFhYaLCPk5MTtFrtffuJiIjA4cOHMWzYMIO+W7Zsed/jdDodLl68iM8++0z/73Do0CGjriEiIuKe/85keUw+qFa8//776NSpE2JjYzF79mxERkbCzs4Ox48fx9mzZ/Hoo48iPj4eHTp0QJ8+fZCamorw8HD8+uuv2LFjB/r27Vul7GttPXr0wLJly9ChQwdotVpMnTrV4K+sxYsXw9fXF9HR0bCzs8OmTZvg4+OjuGrC9u3b8dNPP6Fr165o2LAhdu7cCZ1Oh+bNm1slHldXV7z88suYPHkyvLy8EBAQgNTUVFy/fh0vvPACsrOzH9jHzJkz8fTTTyMgIAADBgyAnZ0dsrOz8d133+HNN9+sURxFRUWYMGEC/vGPf+DkyZN47733sGjRIoSHh+P555/H0KFDsWjRIkRHR+P333/Hvn37EBkZiaeeeuqe/QUFBWHYsGEYOXIkli5diqioKJw/fx4lJSVISkrCmDFjsGbNGgwePBhTpkyBl5cX8vLysHHjxioVtvvx8PDAypUr0aRJEzg7O9/zc7dp0ybExsaic+fO+PTTT3Hs2DGsXbu2xucAgLCwMKSlpSExMRGSJGHGjBlVKgpBQUHIyMjAoEGDoFKp0KhRoyr9TJ48GUlJSYiOjkZ8fDz++9//Ii0tDV999dV9zy9JEtzc3LB69Wr4+vqiqKgIr7/+ulHXMHbsWHTq1AkLFy7Es88+i927d2PXrl1G9UHmwTkfVCtCQkJw6tQpxMfHY9q0aYiKikJsbCzee+89TJo0CXPnzoUkSdi5cye6du2KESNGIDw8HIMGDcL58+fRtGlTa19CFYsWLYK/vz+6dOmCv/3tb5g0aZLBg9Lc3d2RmpqK2NhYtG3bFoWFhdi5c6fJz3qoLZ6enkhLS0OPHj0QERGBlStXYsOGDWjVqpXVYnr77bfRv39/DBkyBDExMcjLy8Pu3bvRsGHDGh3fq1cvbN++HXv27EHbtm3x2GOP4Z133tFXUmpi6NCh+Ouvv9CuXTuMGTMG48aNw0svvQQAWLduHYYOHYqJEyeiefPm6NOnD44fP46AgID79rlixQoMGDAAo0ePRosWLTBq1CiUlZUBuDV34/Dhw9BqtXjiiSfQunVrjB8/Hp6engbDNw8ycOBAZGRkoGvXrmjfvv09P3fJycnYuHEjIiMj8fHHH2PDhg0PrDTcbfHixWjYsCE6duyIxMRE9OrVCzExMQb7zJkzB4WFhQgJCUHjxo3v2U+fPn2wZMkSLFy4EK1atcKqVauwbt06xMXF3ff8kiThn//8J06cOIFHHnkEr732GhYsWGDUNTz22GNYs2YNlixZgqioKOzZswfTp083qg8yDz7hlIjqvbi4OLRp0wbvvvuutUMxO7lPQCWqDcr6k4yIiIhsHpMPIiIisigOuxAREZFFsfJBREREFsXkg4iIiCyKyQcRERFZFJMPIiIisigmH0RERGRRTD6IiIjIoph8EBERkUUx+SAiIiKLYvJBREREFvX/AauW3uvBERE/AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def display_qk_heatmap(qk_per_token):\n",
    "    _, ax = plt.subplots()\n",
    "    im = ax.imshow(qk_per_token.to(float).detach(), cmap='viridis')\n",
    "    ax.set_xticks(range(len(prompt_split_as_tokens)))\n",
    "    ax.set_yticks(range(len(prompt_split_as_tokens)))\n",
    "    ax.set_xticklabels(prompt_split_as_tokens)\n",
    "    ax.set_yticklabels(prompt_split_as_tokens)\n",
    "    ax.figure.colorbar(im, ax=ax)\n",
    "    \n",
    "display_qk_heatmap(qk_per_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., -inf, -inf, -inf, -inf, -inf, -inf, -inf],\n",
       "        [0., 0., -inf, -inf, -inf, -inf, -inf, -inf],\n",
       "        [0., 0., 0., -inf, -inf, -inf, -inf, -inf],\n",
       "        [0., 0., 0., 0., -inf, -inf, -inf, -inf],\n",
       "        [0., 0., 0., 0., 0., -inf, -inf, -inf],\n",
       "        [0., 0., 0., 0., 0., 0., -inf, -inf],\n",
       "        [0., 0., 0., 0., 0., 0., 0., -inf],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = torch.full((len(tokens), len(tokens)), float(\"-inf\"), device=tokens.device)\n",
    "mask = torch.triu(mask, diagonal=1)\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/cUlEQVR4nO3daXgUZfb38V9loROSdELYkowJkR0RYpBFFiVoEKJGAZFlHBZRZ8YNEBHkr7KpRBFUEAREBXUcHZGADqKyCII47AZRMQIGiSOIAyMxIAHT9/OCoR+bzXQ66S6a7+e66jLVVXXXqaQlJ+fcVW0ZY4wAAAD8JCTQAQAAgPMLyQcAAPArkg8AAOBXJB8AAMCvSD4AAIBfkXwAAAC/IvkAAAB+RfIBAAD8KizQAZxvXC6Xvv/+e8XExMiyrECHAwDwkjFGP//8s5KSkhQSUnl/wx85ckRHjx71eZwqVaooIiKiAiKqOCQffvb9998rOTk50GEAAHxUWFioCy64oFLGPnLkiC6sE629+0p9HishIUEFBQW2SkBIPvwsJiZGktS+5f0KC3MEOBrvvPPB/YEOAQACrqioSMnJye5/zyvD0aNHtXdfqQo21ZEzpvzVlaKfXbrw0m919OhRko/z2YlWS1iYQ2Fh9nkjlIXT6Qx0CABgG/5onTtjQnxKPuyK5AMAAJsqNS6V+vDxr6XGVXHBVCCSDwAAbMolI5fKn334cmxlIvkAAMCmXHLJl9qFb0dXnuBrJAEAAFuj8gEAgE2VGqNSU/7WiS/HViaSDwAAbCpY53zQdgEAAH5F5QMAAJtyyag0CCsfJB8AANgUbRcAAIAKQOUDAACbCta7Xah8AABgU64KWLyxatUqZWdnKykpSZZlaeHChR7bLcs67fLkk096dR6SDwAAIEk6dOiQ0tLSNH369NNu37Nnj8fy0ksvybIs3XjjjV6dh7YLAAA2Verj3S7eHpuVlaWsrKwzbk9ISPBYf/vtt9WpUyfVrVvXq/OQfAAAYFOlRj5+qu3x/xYVFXm87nA45HA4fIhM+uGHH/Tuu+/q5Zdf9vpY2i4AANhURc35SE5OVmxsrHvJycnxObaXX35ZMTEx6tGjh9fHUvkAACDIFRYWyul0utd9rXpI0ksvvaSbb75ZERERXh9L8gEAgE25ZKlUlk/HS5LT6fRIPny1evVq5efn6x//+Ee5jif5AADAplzm+OLL8ZXhxRdf1KWXXqq0tLRyHU/yUclKSkpUUlLiXj950g8AAHZRXFysHTt2uNcLCgqUl5en+Ph4paSkSDr+e2zevHmaPHlyuc/DhNNKlpOT4zHJJzk5OdAhAQDOEaX/a7v4snhj48aNSk9PV3p6uiRp2LBhSk9P1+jRo937vPHGGzLGqG/fvuW+LssYmz57NUicrvKRnJysjpc9pLAw7yfpBNKy1Q8GOgQACLiioiLFxsbq4MGDFTqP4nTn+OSLREXHlL9OUPyzS+2a7qnUWMuDtkslq4h7qQEACCYkHwAA2JTLWHIZH+528eHYykTyAQCATZVn3sbJx9sRE04BAIBfUfkAAMCmShWiUh/qBKUVGEtFIvkAAMCmjI9zPgxzPgAAgDeY8wEAAFABqHwAAGBTpSZEpcaHOR82fYwoyQcAADblkiWXD00Kl+yZfdB2AQAAfkXlAwAAmwrWCackHwAA2JTvcz5ouwAAAFD5AADAro5POPXhg+VouwAAAG+4fHy8One7AAAAiMoHAAC2FawTTkk+AACwKZdCgvIhYyQfAADYVKmxVOrDJ9P6cmxlIvkIkLCvv1NYSJVAh+GVqy8dG+gQymXJprGBDgEA8BskHwAA2FSpj3e7lNJ2AQAA3nCZELl8mHDqsumEU261BQAAfkXlAwAAm6LtAgAA/Mol3+5YcVVcKBWKtgsAAPArKh8AANiU7w8Zs2eNgeQDAACb8v3x6vZMPuwZFQAACFpUPgAAsCmXLLnky4RTHq8OAAC8EKxtF5IPAABsyvfnfNgz+bBnVAAAIGhR+QAAwKZcxpLLl4eM+XBsZaLyAQCATbn+13Yp7+Ltcz5WrVql7OxsJSUlybIsLVy48JR9tm3bpuuvv16xsbGKiopSq1attHv3bq/OQ/IBAAAkSYcOHVJaWpqmT59+2u07d+5Uhw4d1LhxY61cuVKfffaZHn74YUVERHh1HtouAADYlMuEyOXDHSveHpuVlaWsrKwzbn/wwQd1zTXXaOLEie7X6tWr53VcVD4AALCpUlk+LxXF5XLp3XffVcOGDdWlSxfVqlVLbdq0OW1r5veQfAAAEOSKioo8lpKSEq/H2Ldvn4qLi/X444+ra9euWrJkibp3764ePXroo48+8mqs8yb52Lt3r4YMGaL69esrIiJCtWvXVvv27TVjxgwdPnw40OEBAHCKE20XXxZJSk5OVmxsrHvJycnxPhaXS5J0ww036N5779Ull1yiBx54QNddd51mzpzp1VjnxZyPb775Ru3bt1dcXJwmTJigZs2ayeFwaOvWrXr++ef1hz/8Qddff32gwwQAwEOp5FPrpPR//y0sLJTT6XS/7nA4vB6rRo0aCgsL00UXXeTxepMmTfTxxx97NdZ5Ufm48847FRYWpo0bN6pXr15q0qSJ6tatqxtuuEHvvvuusrOzJUk//fSTbrvtNtWsWVNOp1NXXnmltmzZ4h5n7NixuuSSS/Tqq68qNTVVsbGx6tOnj37++edAXRoAAL/L6XR6LOVJPqpUqaJWrVopPz/f4/Wvv/5aderU8WqsoK987N+/X0uWLNGECRMUFRV12n0s63hWedNNNykyMlLvvfeeYmNjNWvWLF111VX6+uuvFR8fL+n4bUYLFy7UokWL9N///le9evXS448/rscee+y0Y5eUlHj01oqKiir4CgEAwcrfd7sUFxdrx44d7vWCggLl5eUpPj5eKSkpuv/++9W7d29dccUV6tSpk95//33985//1MqVK706T9BXPnbs2CFjjBo1auTxeo0aNRQdHa3o6GiNHDlSH3/8sdavX6958+apZcuWatCggSZNmqS4uDi99dZb7uNcLpfmzp2riy++WJdffrn69eun5cuXn/H8OTk5Hn225OTkSrtWAEBwOfHBcr4s3ti4caPS09OVnp4uSRo2bJjS09M1evRoSVL37t01c+ZMTZw4Uc2aNdMLL7yg+fPnq0OHDl6dJ+grH2eyfv16uVwu3XzzzSopKdGWLVtUXFys6tWre+z3yy+/aOfOne711NRUxcTEuNcTExO1b9++M55n1KhRGjZsmHu9qKiIBAQAUCZGllw+zPkwXh6bkZEhY8xZ9xk0aJAGDRpU7pik8yD5qF+/vizLOqVHVbduXUlSZGSkpOOlpsTExNOWjuLi4txfh4eHe2yzLMs9A/h0HA5HuXprAAAEq6BPPqpXr67OnTtr2rRpuueee84476NFixbau3evwsLClJqa6t8gAQA4jfK0Tk4+3o7sGVUFe+655/Trr7+qZcuW+sc//qFt27YpPz9ff/vb3/TVV18pNDRUmZmZatu2rbp166YlS5Zo165d+uSTT/Tggw9q48aNgb4EAMB56MSn2vqy2FHQVz6k48+d//TTTzVhwgSNGjVK3333nRwOhy666CINHz5cd955pyzL0uLFi/Xggw/qlltu0Y8//qiEhARdccUVql27dqAvAQCAoGGZ35tZggpVVFSk2NhYXRU/UGEhVQIdjldcKYmBDqFclmwaG+gQAASRE/+OHzx40OPBXZVxjqFrrpcjOvz3DziDkuJjeqb9O5Uaa3mcF5UPAADORb62Tuzadjkv5nwAAAD7oPIBAIBNuRQilw91Al+OrUwkHwAA2FSpsVTqQ+vEl2Mrkz1TIgAAELSofAAAYFPBOuGU5AMAAJsyPn6qrbHpE05JPgAAsKlSWSr14YPlfDm2MtkzJQIAAEGLygcAADblMr7N23DZ9BnmJB8AANiUy8c5H74cW5nsGRUAAAhaVD4AALAplyy5fJg06suxlYnkAwAAm+IJpwAAABWAykeAHOxYT2HhEYEOwyu/VD83c9WmI58OdAjl9sUT9wY6BAABFKwTTkk+AACwKZd8fLy6Ted82DMlAgAAQYvKBwAANmV8vNvF2LTyQfIBAIBN8am2AADAr4J1wqk9owIAAEGLygcAADZF2wUAAPhVsD5enbYLAADwKyofAADYFG0XAADgV8GafNB2AQAAfkXlAwAAmwrWygfJBwAANhWsyQdtFwAA4FckHwAA2JTR/3/WR3kW4+X5Vq1apezsbCUlJcmyLC1cuNBj+8CBA2VZlsfStWtXr6+LtgsAADbl77bLoUOHlJaWpkGDBqlHjx6n3adr166aM2eOe93hcHgdF8kHAAA25e/kIysrS1lZWWfdx+FwKCEhodwxSbRdJEkZGRkaOnRooMMAAKBSFBUVeSwlJSXlHmvlypWqVauWGjVqpDvuuEP79+/3egwqH5Jyc3MVHh4e6DAAAPBQUZWP5ORkj9fHjBmjsWPHej1e165d1aNHD1144YXauXOn/u///k9ZWVn617/+pdDQ0DKPQ/IhKT4+PtAhAABwiopKPgoLC+V0Ot2vl2eehiT16dPH/XWzZs3UvHlz1atXTytXrtRVV11V5nFou8iz7XK62b1xcXGaO3euJOno0aO6++67lZiYqIiICNWpU0c5OTn+DRgAAC84nU6PpbzJx8nq1q2rGjVqaMeOHV4dR+XDS1OnTtU777yjN998UykpKSosLFRhYeEZ9y8pKfHorRUVFfkjTABAEDDGkvGh8uHLsWXx3Xffaf/+/UpMTPTqOJIPL+3evVsNGjRQhw4dZFmW6tSpc9b9c3JyNG7cOD9FBwAIJiee1+HL8d4oLi72qGIUFBQoLy9P8fHxio+P17hx43TjjTcqISFBO3fu1IgRI1S/fn116dLFq/PQdvHSwIEDlZeXp0aNGmnw4MFasmTJWfcfNWqUDh486F7OViUBACCQNm7cqPT0dKWnp0uShg0bpvT0dI0ePVqhoaH67LPPdP3116thw4a69dZbdemll2r16tVet3GofJzEsiwZ4/lMuGPHjrm/btGihQoKCvTee+9p2bJl6tWrlzIzM/XWW2+ddjyHw1FhvTUAwPnF38/5yMjIOOV34G998MEH5Y7lt0g+TlKzZk3t2bPHvb59+3YdPnzYYx+n06nevXurd+/e6tmzp7p27aoDBw5w1wwAoELZfc5HeZF8nOTKK6/UtGnT1LZtW5WWlmrkyJEezwB56qmnlJiYqPT0dIWEhGjevHlKSEhQXFxc4IIGAOAcQvJxksmTJ+uWW27R5ZdfrqSkJE2ZMkWbNm1yb4+JidHEiRO1fft2hYaGqlWrVlq8eLFCQpg+AwCoWP5uu/gLyYeOPyr2hKSkpFN6Wj/99JP769tvv1233367nyIDAJzPaLsAAAC/Mj5WPuyafNArAAAAfkXlAwAAmzKSznLna5mOtyOSDwAAbMolS5Yfn3DqL7RdAACAX1H5AADAprjbBQAA+JXLWLKC8DkftF0AAIBfUfkAAMCmjPHxbheb3u5C8gEAgE0F65wP2i4AAMCvqHwAAGBTwVr5IPkAAMCmgvVuF5IPAABsKlgnnDLnAwAA+BWVDwAAbOp45cOXOR8VGEwFIvkIkIN1QxXqCA10GF4pqe4KdAjlUvV7e/Y8y2JmfsdAh1Auf230UaBDAIJCsE44pe0CAAD8isoHAAA2Zf63+HK8HZF8AABgU7RdAAAAKgCVDwAA7CpI+y4kHwAA2JWPbRfZtO1C8gEAgE3xhFMAAIAKQOUDAACbCta7XUg+AACwK2P5Nm/DpskHbRcAAOBXVD4AALCpYJ1wSvIBAIBdBelzPmi7AAAAv6LyAQCATQXr3S5UPgAAsDPjw+KlVatWKTs7W0lJSbIsSwsXLjzjvn/9619lWZaeeeYZr89D8gEAACRJhw4dUlpamqZPn37W/RYsWKC1a9cqKSmpXOeh7VJGGRkZuuSSS8qV4QEAUB7+brtkZWUpKyvrrPv8+9//1j333KMPPvhA1157bbniIvkoo9zcXIWHhwc6DADA+aSC7nYpKiryeNnhcMjhcHg9nMvlUr9+/XT//feradOm5Q6LtksZxcfHKyYmJtBhAADOK1YFLFJycrJiY2PdS05OTrmieeKJJxQWFqbBgwf7clEkH2WVkZGhoUOHSpKee+45NWjQQBEREapdu7Z69uwZ2OAAADiLwsJCHTx40L2MGjXK6zE2bdqkKVOmaO7cubIs3+6ioe3ipY0bN2rw4MF69dVX1a5dOx04cECrV68+4/4lJSUqKSlxr59c+gIA4IwqqO3idDrldDp9CmX16tXat2+fUlJS3K+Vlpbqvvvu0zPPPKNdu3aVeSySDy/t3r1bUVFRuu666xQTE6M6deooPT39jPvn5ORo3LhxfowQABA0bPSE0379+ikzM9PjtS5duqhfv3665ZZbvBqL5MNLnTt3Vp06dVS3bl117dpVXbt2Vffu3VW1atXT7j9q1CgNGzbMvV5UVKTk5GR/hQsAQJkVFxdrx44d7vWCggLl5eUpPj5eKSkpql69usf+4eHhSkhIUKNGjbw6D3M+vBQTE6PNmzfr9ddfV2JiokaPHq20tDT99NNPp93f4XC4y10VUfYCAJxHjOX74oWNGzcqPT3dXdEfNmyY0tPTNXr06Aq9LCof5RAWFqbMzExlZmZqzJgxiouL04cffqgePXoEOjQAQBDx96faZmRkyHhxkDfzPH6L5MNLixYt0jfffKMrrrhC1apV0+LFi+VyubwuOQEAcL4i+fBSXFyccnNzNXbsWB05ckQNGjTQ66+/7tPDVgAAOC0bTTitSCQfZbRy5crTfg0AQKUpx7yNU463ISacAgAAv6LyAQCATVnm+OLL8XZE8gEAgF0x5wMAAPgVcz4AAAB8R+UDAAC7ou0CAAD8KkiTD9ouAADAr6h8AABgV0Fa+SD5AADArrjbBQAAwHdUPgAAsCmecAoAAPwrSOd80HYBAAB+RfIBAAD8irYLAAA2ZcnHOR8VFknFIvkIkDr/KFRYiCPQYXilqNUfAh1CuRQnBDqC8vtz7PeBDqFcbll/S6BDKJc5recEOgTAE7faAgAA+I7KBwAAdhWkd7uQfAAAYFdBmnzQdgEAAH5F5QMAAJviCacAAMC/aLsAAAD4jsoHAAB2FaSVD5IPAABsKljnfNB2AQAAfkXlAwAAuwrSx6uTfAAAYFfM+QAAAP7EnA8AAIAKQOUDAAC7ou0CAAD8yse2i12TD9ouAABAkrRq1SplZ2crKSlJlmVp4cKFHtvHjh2rxo0bKyoqStWqVVNmZqbWrVvn9XlIPgAAsCtTAYsXDh06pLS0NE2fPv202xs2bKhp06Zp69at+vjjj5Wamqqrr75aP/74o1fnoe0CAIBd+XnOR1ZWlrKyss64/Y9//KPH+lNPPaUXX3xRn332ma666qoyn4fkAwCAIFdUVOSx7nA45HA4fBrz6NGjev755xUbG6u0tDSvjj2v2i4lJSUaPHiwatWqpYiICHXo0EEbNmyQJK1cuVKWZWn58uVq2bKlqlatqnbt2ik/P99jjLffflstWrRQRESE6tatq3HjxunXX38NxOUAAILcied8+LJIUnJysmJjY91LTk5OuWNatGiRoqOjFRERoaefflpLly5VjRo1vBrjvEo+RowYofnz5+vll1/W5s2bVb9+fXXp0kUHDhxw7/Pggw9q8uTJ2rhxo8LCwjRo0CD3ttWrV6t///4aMmSIvvzyS82aNUtz587VY489FojLAQCgTAoLC3Xw4EH3MmrUqHKP1alTJ+Xl5emTTz5R165d1atXL+3bt8+rMc6b5OPQoUOaMWOGnnzySWVlZemiiy7S7NmzFRkZqRdffNG932OPPaaOHTvqoosu0gMPPKBPPvlER44ckSSNGzdODzzwgAYMGKC6deuqc+fOeuSRRzRr1qwznrekpERFRUUeCwAA/uR0Oj0WX1ouUVFRql+/vi677DK9+OKLCgsL8/g9WhbnTfKxc+dOHTt2TO3bt3e/Fh4ertatW2vbtm3u15o3b+7+OjExUZLcGd2WLVs0fvx4RUdHu5fbb79de/bs0eHDh0973pycHI9SV3JycmVcHgAgGPn5bpfycLlcKikp8eoYJpyeJDw83P21ZR3/NECXyyVJKi4u1rhx49SjR49TjouIiDjteKNGjdKwYcPc60VFRSQgAIAy8fdnuxQXF2vHjh3u9YKCAuXl5Sk+Pl7Vq1fXY489puuvv16JiYn6z3/+o+nTp+vf//63brrpJq/Oc94kH/Xq1VOVKlW0Zs0a1alTR5J07NgxbdiwQUOHDi3TGC1atFB+fr7q169f5vNWxIxiAMB5zI9PKd24caM6derkXj/xx/OAAQM0c+ZMffXVV3r55Zf1n//8R9WrV1erVq20evVqNW3a1KvznDfJR1RUlO644w7df//9io+PV0pKiiZOnKjDhw/r1ltv1ZYtW353jNGjR+u6665TSkqKevbsqZCQEG3ZskWff/65Hn30UT9cBQAAlScjI0PGnDnbyc3NrZDznDfJhyQ9/vjjcrlc6tevn37++We1bNlSH3zwgapVq1am47t06aJFixZp/PjxeuKJJxQeHq7GjRvrtttuq+TIAQDnJT5Y7twXERGhqVOnaurUqadsO122d8kll5zyWpcuXdSlS5dKjRMAAMn/cz785by52wUAANjDeVX5AADgnELbBQAA+BNtFwAAgApA5QMAALui7QIAAPwqSJMP2i4AAMCvqHwAAGBTwTrhlOQDAAC7CtK2C8kHAAB2FaTJB3M+AACAX1H5AADAppjzAQAA/Iu2CwAAgO+ofAAAYFO0XQAAgH8FaduF5CNQXC5JrkBH4ZWY/J8CHUK5xHx2NNAhlNuFrW4NdAjlUnW7I9AhlMtFC58OdAjl8uWEewMdAuAVkg8AAOyKygcAAPAn63+LL8fbEXe7AAAAv6LyAQCAXdF2AQAA/sSttgAAwL+CtPLBnA8AAOBXVD4AALAzm1YvfEHyAQCATQXrnA/aLgAAwK+ofAAAYFdBOuGU5AMAAJui7QIAAFABqHwAAGBXtF0AAIA/0XYBAACoAFQ+AACwqyBtu1D5AADArkwFLF5YtWqVsrOzlZSUJMuytHDhQve2Y8eOaeTIkWrWrJmioqKUlJSk/v376/vvv/f6skg+AACwqRNzPnxZvHHo0CGlpaVp+vTpp2w7fPiwNm/erIcfflibN29Wbm6u8vPzdf3113t9XbRdfGBZlhYsWKBu3boFOhQAAHyWlZWlrKys026LjY3V0qVLPV6bNm2aWrdurd27dyslJaXM5wm65KO0tFSWZSkkhKIOAOAcV0FzPoqKijxedjgccjgcPgx83MGDB2VZluLi4rw6LuC/oTMyMnT33Xfr7rvvVmxsrGrUqKGHH35Yxhz/jpWUlGj48OH6wx/+oKioKLVp00YrV650Hz937lzFxcXpnXfe0UUXXSSHw6Hdu3erpKREI0eOVHJyshwOh+rXr68XX3zRfdznn3+urKwsRUdHq3bt2urXr5/+85//eMQ1ePBgjRgxQvHx8UpISNDYsWPd21NTUyVJ3bt3l2VZ7nUAACqKZYzPiyQlJycrNjbWveTk5Pgc25EjRzRy5Ej17dtXTqfTq2MDnnxI0ssvv6ywsDCtX79eU6ZM0VNPPaUXXnhBknT33XfrX//6l9544w199tlnuummm9S1a1dt377dffzhw4f1xBNP6IUXXtAXX3yhWrVqqX///nr99dc1depUbdu2TbNmzVJ0dLQk6aefftKVV16p9PR0bdy4Ue+//75++OEH9erV65S4oqKitG7dOk2cOFHjx493l5w2bNggSZozZ4727NnjXj9ZSUmJioqKPBYAAPypsLBQBw8edC+jRo3yabxjx46pV69eMsZoxowZXh9vi7ZLcnKynn76aVmWpUaNGmnr1q16+umn1aVLF82ZM0e7d+9WUlKSJGn48OF6//33NWfOHE2YMEHS8W/Cc889p7S0NEnS119/rTfffFNLly5VZmamJKlu3bru802bNk3p6enu4yXppZdeUnJysr7++ms1bNhQktS8eXONGTNGktSgQQNNmzZNy5cvV+fOnVWzZk1JUlxcnBISEs54bTk5ORo3blxFfasAAOeTCmq7OJ1Or6sTZ3Ii8fj222/14YcflmtcW1Q+LrvsMlmW5V5v27attm/frq1bt6q0tFQNGzZUdHS0e/noo4+0c+dO9/5VqlRR8+bN3et5eXkKDQ1Vx44dT3u+LVu2aMWKFR5jNm7cWJI8xv3tmJKUmJioffv2eXVto0aN8sg2CwsLvToeAHD+8vfdLr/nROKxfft2LVu2TNWrVy/XOLaofJxJcXGxQkNDtWnTJoWGhnpsO9FCkaTIyEiP5CUyMvJ3x83OztYTTzxxyrbExET31+Hh4R7bLMuSy+Xy6hoqalIPAACVrbi4WDt27HCvFxQUKC8vT/Hx8UpMTFTPnj21efNmLVq0SKWlpdq7d68kKT4+XlWqVCnzeWyRfKxbt85jfe3atWrQoIHS09NVWlqqffv26fLLLy/zeM2aNZPL5dJHH33kbrv8VosWLTR//nylpqYqLKz834Lw8HCVlpaW+3gAAM7Kz0843bhxozp16uReHzZsmCRpwIABGjt2rN555x1J0iWXXOJx3IoVK5SRkVHm89ii7bJ7924NGzZM+fn5ev311/Xss89qyJAhatiwoW6++Wb1799fubm5Kigo0Pr165WTk6N33333jOOlpqZqwIABGjRokBYuXKiCggKtXLlSb775piTprrvu0oEDB9S3b19t2LBBO3fu1AcffKBbbrnFq2QiNTVVy5cv1969e/Xf//7X5+8DAAC/5e+2S0ZGhowxpyxz585VamrqabcZY7xKPCSbJB/9+/fXL7/8otatW+uuu+7SkCFD9Oc//1nS8btJ+vfvr/vuu0+NGjVSt27dtGHDht99mMmMGTPUs2dP3XnnnWrcuLFuv/12HTp0SJKUlJSkNWvWqLS0VFdffbWaNWumoUOHKi4uzqvng0yePFlLly5VcnKy0tPTy/8NAADgPGKZEw/UCJCMjAxdcskleuaZZwIZht8UFRUpNjZWmUl/UVjIuTUXxMTFBDqE8ik5GugIym3bg/GBDqFcqm4/t97bJ1jeTemyjS8n3BvoEM4rJ/4dP3jwYIXdQXKmc7To85hCq0SUe5zSo0e0+Y0HKzXW8rDFnA8AAHAqX+9Yqei7XSoKyQcAAHbl5wmn/hLw5OO3j0oHAADBL+DJBwAAODO7tk58QfIBAIBdGXN88eV4G7LFrbYAAOD8QeUDAACb4m4XAADgX0F6twttFwAA4FdUPgAAsCnL5duTd+361F6SDwAA7Iq2CwAAgO+ofAAAYFPc7QIAAPwrSB8yRvIBAIBNUflAhfp1zw+SFR7oMLwS+nNxoEMol9KiokCHUG5J79UOdAjlEr3r50CHUC4lNSICHUK5XHXFY4EOodyWr3ow0CEgAEg+AACwqyC924XkAwAAmwrWtgu32gIAAL+i8gEAgF1xtwsAAPAn2i4AAAAVgMoHAAB2xd0uAADAn2i7AAAAVAAqHwAA2JXLHF98Od6GSD4AALAr5nwAAAB/suTjnI8Ki6RiMecDAAD4FZUPAADsiiecAgAAf+JWWwAAgApA5QMAALsK0rtdqHwAAGBTljE+L95YtWqVsrOzlZSUJMuytHDhQo/tubm5uvrqq1W9enVZlqW8vLxyXZdXyUdGRoaGDh1arhN5a9euXT5d2O+ZO3eu4uLiKmVsAADORYcOHVJaWpqmT59+xu0dOnTQE0884dN5vGq75ObmKjw83KcTllVycrL27NmjGjVq+DxWamqqhg4d6pE49e7dW9dcc43PYwMAUGlc/1t8Od4LWVlZysrKOuP2fv36STpeIPCFV8lHfHy8TyfzRmhoqBISEs643Rij0tJShYWVb9pKZGSkIiMjyxseAACVrjytk5OPt6Nyt11SU1M1YcIEDRo0SDExMUpJSdHzzz/v3vfo0aO6++67lZiYqIiICNWpU0c5OTnu7ZZlacaMGcrKylJkZKTq1q2rt956y7395LbLypUrZVmW3nvvPV166aVyOBz6+OOPtXPnTt1www2qXbu2oqOj1apVKy1btswj5m+//Vb33nuvLMuSZR1/3tvp2i4zZsxQvXr1VKVKFTVq1Eivvvqqx3bLsvTCCy+oe/fuqlq1qho0aKB33nnHm28hAAB+V1RU5LGUlJQENB6fJpxOnjxZLVu21Keffqo777xTd9xxh/Lz8yVJU6dO1TvvvKM333xT+fn5eu2115Samupx/MMPP6wbb7xRW7Zs0c0336w+ffpo27ZtZz3nAw88oMcff1zbtm1T8+bNVVxcrGuuuUbLly/Xp59+qq5duyo7O1u7d++WdLxVdMEFF2j8+PHas2eP9uzZc9pxFyxYoCFDhui+++7T559/rr/85S+65ZZbtGLFCo/9xo0bp169eumzzz7TNddco5tvvlkHDhw4Y7wlJSWn/NABACgTUwGLjk9liI2NdS+/LQYEgk+32l5zzTW68847JUkjR47U008/rRUrVqhRo0bavXu3GjRooA4dOsiyLNWpU+eU42+66SbddtttkqRHHnlES5cu1bPPPqvnnnvujOccP368Onfu7F6Pj49XWlqae/2RRx7RggUL9M477+juu+9WfHy8QkNDFRMTc9Y2zqRJkzRw4ED39QwbNkxr167VpEmT1KlTJ/d+AwcOVN++fSVJEyZM0NSpU7V+/Xp17dr1tOPm5ORo3LhxZzwvAABnVEFPOC0sLJTT6XS/7HA4fI3MJz5VPpo3b+7+2rIsJSQkaN++fZKO/5LOy8tTo0aNNHjwYC1ZsuSU49u2bXvK+u9VPlq2bOmxXlxcrOHDh6tJkyaKi4tTdHS0tm3b5q58lNW2bdvUvn17j9fat29/Sjy/veaoqCg5nU73NZ/OqFGjdPDgQfdSWFjoVVwAgPPXiSec+rJIktPp9FgCnXz4VPk4+c4Xy7Lkch2fWtuiRQsVFBTovffe07Jly9SrVy9lZmZ6zOsoj6ioKI/14cOHa+nSpZo0aZLq16+vyMhI9ezZU0ePHvXpPGdytms+HYfDEfAfMgAAZVFcXKwdO3a41wsKCpSXl6f4+HilpKTowIED2r17t77//ntJck+1SEhIOGt34WSV+pAxp9Op3r17a/bs2frHP/6h+fPne8yPWLt2rcf+a9euVZMmTbw6x5o1azRw4EB1795dzZo1U0JCwim3AFWpUkWlpaVnHadJkyZas2bNKWNfdNFFXsUDAECFOdF28WXxwsaNG5Wenq709HRJx6cgpKena/To0ZKkd955R+np6br22mslSX369FF6erpmzpzp1Xkq7fHqTz31lBITE5Wenq6QkBDNmzdPCQkJHneYzJs3Ty1btlSHDh302muvaf369XrxxRe9Ok+DBg2Um5ur7OxsWZalhx9++JRKRGpqqlatWqU+ffrI4XCc9tkh999/v3r16qX09HRlZmbqn//8p3Jzcz3unAEAwJ8s1/HFl+O9kZGRIXOWhGXgwIEaOHBg+QP6n0qrfMTExGjixIlq2bKlWrVqpV27dmnx4sUKCfn/pxw3bpzeeOMNNW/eXK+88opef/11rysNTz31lKpVq6Z27dopOztbXbp0UYsWLTz2GT9+vHbt2qV69eqpZs2apx2nW7dumjJliiZNmqSmTZtq1qxZmjNnjjIyMry+dgAAcGaWOVuKU5kntiwtWLBA3bp1C8TpA6aoqEixsbHKsLopzPLP02IrSmhMTKBDKJfSc/j25uJelwU6hHKJ3nUo0CGUS0mNiECHUC7hRccCHUK5LV/1YKBD8NqJf8cPHjzocQdJZZwjo/WDCgsr//vy11+PaOX6xyo11vLgU20BALArPtUWAADAdwGrfASo2wMAwDkjWD/bhbYLAAB2VUFPOLUb2i4AAMCvqHwAAGBXRpIPz/mw64RTkg8AAGyKOR8AAMC/jHyc81FhkVQo5nwAAAC/ovIBAIBdBendLiQfAADYlUuS5ePxNkTbBQAA+BWVDwAAbIq7XQAAgH8F6ZwP2i4AAMCvqHwAAGBXQVr5IPkIkO/vba1QR0Sgw/COPd/Dv8u5y6bTvcsg6s//DnQI5bLrXxcEOoRyCf/Zl9sKAifkWHigQyi3B7bcGOgQvFZSfMx/JwvS5IO2CwAA8CsqHwAA2FWQPueD5AMAAJviVlsAAOBfzPkAAADwHZUPAADsymUky4fqhcuelQ+SDwAA7Iq2CwAAgO+ofAAAYFs+Vj5s+nRIkg8AAOyKtgsAAIDvqHwAAGBXLiOfWifc7QIAALxiXMcXX463IdouAADAr6h8AABgV0E64ZTkAwAAu2LOBwAA8KsgrXww58NHc+fOVVxcXKDDAADAZ6tWrVJ2draSkpJkWZYWLlzosd0Yo9GjRysxMVGRkZHKzMzU9u3bvT4PyQcAAHZl9P+rH+VavDvdoUOHlJaWpunTp592+8SJEzV16lTNnDlT69atU1RUlLp06aIjR454dR7aLgAA2JWf2y5ZWVnKyso6w1BGzzzzjB566CHdcMMNkqRXXnlFtWvX1sKFC9WnT58yn+e8qny8//776tChg+Li4lS9enVdd9112rlzpyRp165dsixLubm56tSpk6pWraq0tDT961//8hhj7ty5SklJUdWqVdW9e3ft378/EJcCAECZFRUVeSwlJSVej1FQUKC9e/cqMzPT/VpsbKzatGlzyu/K33NeJR+HDh3SsGHDtHHjRi1fvlwhISHq3r27XK7//xCWBx98UMOHD1deXp4aNmyovn376tdff5UkrVu3Trfeeqvuvvtu5eXlqVOnTnr00UfPes6SkpJTfugAAJSJy+X7Iik5OVmxsbHuJScnx+tQ9u7dK0mqXbu2x+u1a9d2byur86rtcuONN3qsv/TSS6pZs6a+/PJLRUdHS5KGDx+ua6+9VpI0btw4NW3aVDt27FDjxo01ZcoUde3aVSNGjJAkNWzYUJ988onef//9M54zJydH48aNq6QrAgAEtQpquxQWFsrpdLpfdjgcvkbmk/Oq8rF9+3b17dtXdevWldPpVGpqqiRp9+7d7n2aN2/u/joxMVGStG/fPknStm3b1KZNG48x27Zte9Zzjho1SgcPHnQvhYWFFXEpAACUmdPp9FjKk3wkJCRIkn744QeP13/44Qf3trI6r5KP7OxsHThwQLNnz9a6deu0bt06SdLRo0fd+4SHh7u/tixLkjzaMt5yOByn/NABACgTn+508bFqcpILL7xQCQkJWr58ufu1oqIirVu37nf/ED/ZedN22b9/v/Lz8zV79mxdfvnlkqSPP/7YqzGaNGniTlhOWLt2bYXFCACABz8/4bS4uFg7duxwrxcUFCgvL0/x8fFKSUnR0KFD9eijj6pBgwa68MIL9fDDDyspKUndunXz6jznTfJRrVo1Va9eXc8//7wSExO1e/duPfDAA16NMXjwYLVv316TJk3SDTfcoA8++OCs8z0AADiXbNy4UZ06dXKvDxs2TJI0YMAAzZ07VyNGjNChQ4f05z//WT/99JM6dOig999/XxEREV6d57xpu4SEhOiNN97Qpk2bdPHFF+vee+/Vk08+6dUYl112mWbPnq0pU6YoLS1NS5Ys0UMPPVRJEQMAznfGuHxevJGRkSFjzCnL3LlzJR2fjjB+/Hjt3btXR44c0bJly9SwYUOvr+u8qXxIUmZmpr788kuP18xv+mHmpN5YXFzcKa8NGjRIgwYN8njtvvvuq+BIAQDQ8Tkbvnw4nE0/2+W8Sj4AADinGB/nfNg0+Thv2i4AAMAeqHwAAGBXLpdklf9xD/Jyzoe/kHwAAGBXtF0AAAB8R+UDAACbMi6XjA9tF29vtfUXkg8AAOyKtgsAAIDvqHwAAGBXLiNZwVf5IPkAAMCujJHky6229kw+aLsAAAC/ovIBAIBNGZeR8aHtcvLnk9kFyQcAAHZlXPKt7cKttgAAwAvBWvlgzgcAAPArKh9+diILLS05EuBIysGeCfTv+vWYPcuOZfHroZJAh1AuriPn4PtbUmmJFegQysUcC3QE5VdSfO4FX3LoeMz+qCr8akp8ap38Knt+fy1j15pMkPruu++UnJwc6DAAAD4qLCzUBRdcUCljHzlyRBdeeKH27t3r81gJCQkqKChQREREBURWMUg+/Mzlcun7779XTEyMLKti/8oqKipScnKyCgsL5XQ6K3Tsynauxk7c/kXc/neuxl6ZcRtj9PPPPyspKUkhIZU3e+HIkSM6evSoz+NUqVLFVomHRNvF70JCQiotUz7B6XSeU/9I/Na5Gjtx+xdx+9+5GntlxR0bG1vhY54sIiLCdklDRWHCKQAA8CuSDwAA4FckH0HE4XBozJgxcjgcgQ7Fa+dq7MTtX8Ttf+dq7Odq3OcLJpwCAAC/ovIBAAD8iuQDAAD4FckHAADwK5IP4CwyMjI0dOjQQIdRIYLpWoKdP39Wu3btkmVZysvLq5Tx586dq7i4uEoZuyLYPb5gxYRTVJq9e/cqJydH7777rr777jvFxsaqfv36+tOf/qQBAwaoatWqgQ7xdx04cEDh4eGKiYkJdCg+C6ZrCXbe/Kwsy9KCBQvUrVu3cp2rtLRUP/74o2rUqKGwMN+eO5mamqqhQ4d6JE6//PKLfv75Z9WqVcunsSvL3LlzNXToUP3000+BDuW8whNOUSm++eYbtW/fXnFxcZowYYKaNWsmh8OhrVu36vnnn9cf/vAHXX/99YEO83fFx8cHOoQKE0zXYnelpaWyLKvcj972588qNDRUCQkJZ9xujFFpaWm5E5PIyEhFRkaWNzwEKwNUgi5dupgLLrjAFBcXn3a7y+Uyxhjz3//+19x6662mRo0aJiYmxnTq1Mnk5eW59xszZoxJS0szr7zyiqlTp45xOp2md+/epqioyC/X0bFjRzNkyBBjjDGSzIIFCzy2x8bGmjlz5hhjjCkpKTF33XWXSUhIMA6Hw6SkpJgJEyb4Jc6y+O21TJ8+3dSvX984HA5Tq1Ytc+ONNwY0tiNHjph77rnH1KxZ0zgcDtO+fXuzfv16Y4wxK1asMJLMsmXLzKWXXmoiIyNN27ZtzVdffeUxxsKFC016erpxOBzmwgsvNGPHjjXHjh0r0/k7duxo7rrrLnPXXXcZp9Npqlevbh566CH3+/TIkSPmvvvuM0lJSaZq1aqmdevWZsWKFe7j58yZY2JjY83bb79tmjRpYkJDQ01BQYE5cuSIGTFihLngggtMlSpVTL169cwLL7zgPm7r1q2ma9euJioqytSqVcv86U9/Mj/++KP7Z9WxY0cTExNjLr/8clOlShVjWZZxOp1m1qxZxhhj6tSpY3T886bdy8nvO0nmueeeM127djURERHmwgsvNPPmzXNvLygoMJLMp59+6vH9Xrx4sWnRooUJDw83K1asMDt27DDXX3+9qVWrlomKijItW7Y0S5cu9fgenhzLb783v/Xcc8+ZunXrmvDwcNOwYUPzyiuveGyXZGbPnm26detmIiMjTf369c3bb79t3nvvPdO+fXsTGxtr4uPjzbXXXmt27NjhcR3z5883GRkZJjIy0jRv3tx88sknHmPPmTPHJCcnm8jISNOtWzczadKkU+JD5SP5QIX7z3/+YyzLMjk5Ob+7b2ZmpsnOzjYbNmwwX3/9tbnvvvtM9erVzf79+40xx5OP6Oho06NHD7N161azatUqk5CQYP7v//6vsi/DGONd8vHkk0+a5ORks2rVKrNr1y6zevVq8/e//90vcZbFiWvZsGGDCQ0NNX//+9/Nrl27zObNm82UKVMCGtvgwYNNUlKSWbx4sfniiy/MgAEDTLVq1cz+/fvdvwzbtGljVq5cab744gtz+eWXm3bt2rmPX7VqlXE6nWbu3Llm586dZsmSJSY1NdWMHTu2TOfv2LGjiY6ONkOGDDFfffWV+dvf/maqVq1qnn/+eWOMMbfddptp166dWbVqldmxY4d58sknjcPhMF9//bUx5vgvtPDwcNOuXTuzZs0a89VXX5lDhw6ZXr16meTkZJObm2t27txpli1bZt544w1jzPHEu2bNmmbUqFFm27ZtZvPmzaZz586mU6dOHsmHZVkmMjLSjBkzxkycONFIMiEhIearr74y+/btM5JMfHy8WbBggdm0adMp7ztJpnr16mb27NkmPz/fPPTQQyY0NNR8+eWXxpgzJx/Nmzc3S5YsMTt27DD79+83eXl5ZubMmWbr1q3m66+/Ng899JCJiIgw3377rTHGmP3795sLLrjAjB8/3uzZs8fs2bPH/b357S/33NxcEx4ebqZPn27y8/PN5MmTTWhoqPnwww89Yr7gggvM3//+d7N9+3YzePBgEx0dbebMmWPmz59vtm/fbj799FOTnZ1tmjVrZkpLS93X0bhxY7No0SKTn59vevbsaerUqeNOQteuXWtCQkLME088YfLz882UKVNMXFwcyUcAkHygwq1du9ZIMrm5uR6vV69e3URFRZmoqCgzYsQIs3r1auN0Os2RI0c89qtXr577L7sxY8aYqlWrelQ67r//ftOmTZvKvxDjXfJxzz33mCuvvNL917LdnLiW+fPnG6fT6bfq0e8pLi424eHh5rXXXnO/dvToUZOUlGQmTpzoUfk44d133zWSzC+//GKMMeaqq646pcr06quvmsTExDLF0LFjR9OkSROPn93IkSNNkyZNzLfffmtCQ0PNv//9b49jrrrqKjNq1ChjzPFfsJI8qnb5+flGkkd14LceeeQRc/XVV3u8VlhYaCSZ1q1bu5MPh8Nh/vSnP7n3admypalataqZMWOGMeb4+7JZs2ZnfN9JMn/96189XmvTpo254447jDFnTj4WLlx4xu/XCU2bNjXPPvuse71OnTrm6aef9tjn5OSjXbt25vbbb/fY56abbjLXXHONR8wPPfSQe724uNhIMu+9957HcT/++KORZLZu3eq+jt9Wlr744gsjyWzbts0YY0zfvn09zmOMMb179yb5CADudoHfrF+/Xnl5eWratKlKSkq0ZcsWFRcXq3r16oqOjnYvBQUF2rlzp/u41NRUj4l3iYmJ2rdvXyAu4awGDhyovLw8NWrUSIMHD9aSJUsCHdJpde7cWXXq1FHdunXVr18/vfbaazp8+HDA4tm5c6eOHTum9u3bu18LDw9X69attW3bNvdrzZs3d3+dmJgoSe73wZYtWzR+/HiP99Htt9+uPXv2lPnaLrvsMlmW5V5v27attm/frq1bt6q0tFQNGzb0GP+jjz7yeJ9WqVLFI8a8vDyFhoaqY8eOpz3fli1btGLFCo8xGzduLOn4JM0zjZuUlCSHw+Hx/0BBQcFZ33dt27Y9Zf2339vTadmypcd6cXGxhg8friZNmiguLk7R0dHatm2bdu/efdZxTrZt2zaPn7UktW/f/pR4fnvNUVFRcjqd+uyzz9S3b1/VrVtXTqdTqampkuQRw9neJ9u2bVObNm08znPy9wb+wYRTVLj69evLsizl5+d7vF63bl1Jck8+Ky4uVmJiolauXHnKGL+99S08PNxjm2VZcrlcFRt0GViWJXPSzWHHjh1zf92iRQsVFBTovffe07Jly9SrVy9lZmbqrbfe8neoZxUTE6PNmzdr5cqVWrJkiUaPHq2xY8dqw4YNtr7l8LfvgxNJwon3QXFxscaNG6cePXqccpyvH0leXFys0NBQbdq0SaGhoR7boqOj3V9HRkZ6JC+/N8myuLhY2dnZeuKJJ07Zdsstt3isn+3aJWnWrFkKDQ2t0PddVFSUx/rw4cO1dOlSTZo0SfXr11dkZKR69uypo0eP+nSeMznd//dPP/20mjdvrtmzZyspKUkul0sXX3yxRwy/972CPZB8oMJVr15dnTt31rRp03TPPfec8o/YCS1atNDevXsVFhbm/gvGzmrWrKk9e/a417dv337KX9VOp1O9e/dW79691bNnT3Xt2lUHDhyw3Z0mYWFhyszMVGZmpsaMGaO4uDh9+OGHp/3lXdnq1aunKlWqaM2aNapTp46k40ndhg0byvysixYtWig/P1/169cvdxzr1q3zWF+7dq0aNGig9PR0lZaWat++fbr88svLPF6zZs3kcrn00UcfKTMz87Qxz58/X6mpqafcSXJyknM24eHhcjgcuvHGG8/4vlu7dq369+/vcW3p6ellPockrVmzRgMHDlT37t0lHU+edu3a5bFPlSpVVFpaetZxmjRpojVr1mjAgAEeY1900UVnPc7lcmnv3r1688033T+Hjz/+2KtraNKkyWl/zvA/kg9Uiueee07t27dXy5YtNXbsWDVv3lwhISHasGGDvvrqK1166aXKzMxU27Zt1a1bN02cOFENGzbU999/r3fffVfdu3c/pewbaFdeeaWmTZumtm3bqrS0VCNHjvT4K+upp55SYmKi0tPTFRISonnz5ikhIcF21YRFixbpm2++0RVXXKFq1app8eLFcrlcatSoUUDiiYqK0h133KH7779f8fHxSklJ0cSJE3X48GHdeuut2rJly++OMXr0aF133XVKSUlRz549FRISoi1btujzzz/Xo48+WqY4du/erWHDhukvf/mLNm/erGeffVaTJ09Ww4YNdfPNN6t///6aPHmy0tPT9eOPP2r58uVq3ry5rr322tOOl5qaqgEDBmjQoEGaOnWq0tLS9O2332rfvn3q1auX7rrrLs2ePVt9+/bViBEjFB8frx07duiNN944pcJ2NrGxsZo5c6Zq1aqliIiI077v5s2bp5YtW6pDhw567bXXtH79er344otlPockNWjQQLm5ucrOzpZlWXr44YdPqSikpqZq1apV6tOnjxwOh2rUqHHKOPfff7969eql9PR0ZWZm6p///Kdyc3O1bNmys57fsixFR0fr+eefV2Jionbv3q0HHnjAq2sYPHiw2rdvr0mTJumGG27QBx98oPfff9+rMVAxmPOBSlGvXj19+umnyszM1KhRo5SWlqaWLVvq2Wef1fDhw/XII4/IsiwtXrxYV1xxhW655RY1bNhQffr00bfffqvatWsH+hJOMXnyZCUnJ+vyyy/XH//4Rw0fPtzjQWkxMTGaOHGiWrZsqVatWmnXrl1avHhxuZ/1UFni4uKUm5urK6+8Uk2aNNHMmTP1+uuvq2nTpgGL6fHHH9eNN96ofv36qUWLFtqxY4c++OADVatWrUzHd+nSRYsWLdKSJUvUqlUrXXbZZXr66afdlZSy6N+/v3755Re1bt1ad911l4YMGaI///nPkqQ5c+aof//+uu+++9SoUSN169ZNGzZsUEpKylnHnDFjhnr27Kk777xTjRs31u23365Dhw5JOj53Y82aNSotLdXVV1+tZs2aaejQoYqLi/No3/ye3r17a9WqVbriiivUpk2b077vxo0bpzfeeEPNmzfXK6+8otdff/13Kw0ne+qpp1StWjW1a9dO2dnZ6tKli1q0aOGxz/jx47Vr1y7Vq1dPNWvWPO043bp105QpUzRp0iQ1bdpUs2bN0pw5c5SRkXHW81uWpb/+9a/atGmTLr74Yt1777168sknvbqGyy67TLNnz9aUKVOUlpamJUuW6KGHHvJqDFQMnnAK4LyXkZGhSy65RM8880ygQ6lwvj4BFagM9vqTDAAABD2SDwAA4Fe0XQAAgF9R+QAAAH5F8gEAAPyK5AMAAPgVyQcAAPArkg8AAOBXJB8AAMCvSD4AAIBfkXwAAAC/IvkAAAB+9f8A2yKCioixmZUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "qk_per_token_after_masking = qk_per_token + mask\n",
    "display_qk_heatmap(qk_per_token_after_masking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGiCAYAAAA1LsZRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABAlklEQVR4nO3de1xUdf7H8fcBYUAR8AqoKOv9ijfS8JLWYrqVpWWa26ZZuVtpamaav7xXUqam3bQstdrKsuyyaWa50cUyTcWslNQ02FY0c5XwAjbz/f3hOusIKjDAmWFez8fj+3gwZ873fD8Hp+bD5/s951jGGCMAAACbBNkdAAAACGwkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAwFYkIwAAQJL06aefqm/fvqpTp44sy9Lbb799wT5paWnq0KGDHA6HGjdurKVLlxZ7XJIRAAAgSTp69Kjatm2rp556qkj779mzR1deeaUuvfRSpaena8yYMbrtttv0wQcfFGtciwflAQCAs1mWpbfeekv9+vU75z4TJkzQypUr9e2337q33XDDDTp8+LBWr15d5LEqeRMois/lcunf//63qlatKsuy7A4HAFBMxhj99ttvqlOnjoKCym6C4cSJE8rPz/f6OMaYAt83DodDDofD62N/+eWXSklJ8djWu3dvjRkzpljHIRkpZ//+978VHx9vdxgAAC9lZWWpXr16ZXLsEydO6A8NIpR9wOn1sSIiIpSbm+uxberUqZo2bZrXx87OzlZMTIzHtpiYGOXk5Oj48eMKDw8v0nFIRspZ1apVJUk/bU5QZIR/Ldnp37SN3SEAgO1+10l9rlXu/5+Xhfz8fGUfcGrPpgaKrFry74qc31z6Q8eflJWVpcjISPf20qiKlCaSkXJ2ulQWGRHk1QfMDpWsELtDAAD7/XelZXlMtUdWLZ3visjISI9kpLTExsZq//79Htv279+vyMjIIldFJJIRAAB8ltO45PTiMhOncZVeMIVITk7WqlWrPLZ9+OGHSk5OLtZx/OtPcwAAAohLxutWHLm5uUpPT1d6erqkU5fupqenKzMzU5I0ceJEDRkyxL3/7bffrh9//FHjx4/Xjh079PTTT+v111/X3XffXaxxqYwAAOCjXHLJm9pGcXt//fXXuvTSS92vx44dK0kaOnSoli5dqn379rkTE0n6wx/+oJUrV+ruu+/W/PnzVa9ePT333HPq3bt3scYlGQEAAJKknj176ny3Hyvs7qo9e/bUli1bvBqXZAQAAB/lNEZOL+5N6k3f8kQyAgCAjyrJuo+z+/sDFrACAABbURkBAMBHuWTkDIDKCMkIAAA+imkaAACAckBlBAAAH8XVNAAAwFau/zZv+vsDpmkAAICtqIwAAOCjnF5eTeNN3/JEMgIAgI9yGnn51N7Si6UskYwAAOCjWDMCAABQDqiMAADgo1yy5JTlVX9/QDICAICPcplTzZv+/oBkpIzl5eUpLy/P/TonJ8fGaAAA8D2sGSljqampioqKcrf4+Hi7QwIA+Annf6dpvGn+gGSkjE2cOFFHjhxxt6ysLLtDAgD4iUBJRpimKWMOh0MOh8PuMAAA8FkkIwAA+CiXseQyXlxN40Xf8kQyAgCAj/J2qsVfpmlYMwIAAGxFZQQAAB/lVJCcXtQNnKUYS1kiGQEAwEcZL9eMGNaMAAAAb7BmBAAAoBxQGQEAwEc5TZCcxos1IzybBgAAeMMlSy4vJjFc8o9shGkaAABgKyojAAD4qEBZwEoyAgCAj/J+zQjTNAAAABdEZQQAAB91agGrFw/KY5oGAAB4w+Xl7eC5mgYAAKAIqIwAAOCjAmUBK8kIAAA+yqWggLjpGckIAAA+ymksOb148q43fcsTyYhNejx0q4JDw+wOo1gcq/fbHUKJRPT50e4QAADnQTICAICPcnp5NY2TaRoAAOANlwmSy4sFrC4/WcDKpb0AAMBWVEYAAPBRTNMAAABbueTdFTGu0gulTDFNAwAAbEVlBAAAH+X9Tc/8o+ZAMgIAgI/y/nbw/pGM+EeUAACgwqIyAgCAj3LJkkveLGDldvAAAMALgTJNQzICAICP8v4+I/6RjPhHlAAAoMKiMgIAgI9yGUsub2565kXf8kQyAgCAj3J5OU3jL/cZ8Y8oAQBAhUVlBAAAH+UyQXJ5cUWMN33LE8kIAAA+yilLTi/uFeJN3/LkHykTAACosAImGcnOztbo0aPVuHFjhYWFKSYmRl27dtWCBQt07Ngxu8MDAKCA09M03jR/EBDTND/++KO6du2q6OhozZw5U23atJHD4dC2bdv07LPPqm7durr66qvtDhMAAA9OeTfV4iy9UMqUf6RMXrrzzjtVqVIlff311xo4cKBatGihhg0b6pprrtHKlSvVt29fSdLhw4d12223qVatWoqMjNRll12mrVu3uo8zbdo0tWvXTi+99JISEhIUFRWlG264Qb/99ptdpwYAgN+r8MnIr7/+qjVr1mjEiBGqUqVKoftY1qms8/rrr9eBAwf0/vvva9OmTerQoYP++Mc/6tChQ+59d+/erbffflvvvfee3nvvPX3yySd6+OGHzzl+Xl6ecnJyPBoAAEURKNM0/hGlF3bt2iVjjJo1a+axvWbNmoqIiFBERIQmTJigzz//XBs2bNDy5cuVlJSkJk2aaPbs2YqOjtYbb7zh7udyubR06VK1bt1a3bt310033aS1a9eec/zU1FRFRUW5W3x8fJmdKwCgYjn9oDxvmj/wjyjLwIYNG5Senq5WrVopLy9PW7duVW5urmrUqOFOUiIiIrRnzx7t3r3b3S8hIUFVq1Z1v46Li9OBAwfOOc7EiRN15MgRd8vKyirT8wIAVBxGllxeNFPC9SZPPfWUEhISFBYWps6dO2vDhg3n3X/evHlq1qyZwsPDFR8fr7vvvlsnTpwo8ngVfgFr48aNZVmWMjIyPLY3bNhQkhQeHi5Jys3NVVxcnNLS0gocIzo62v1zSEiIx3uWZcnlcp1zfIfDIYfDUcLoAQAoX6+99prGjh2rhQsXqnPnzpo3b5569+6tjIwM1a5du8D+r7zyiu677z4tXrxYXbp00Q8//KCbb75ZlmVp7ty5RRqzwldGatSooV69eunJJ5/U0aNHz7lfhw4dlJ2drUqVKqlx48YerWbNmuUYMQAAp9gxTTN37lwNHz5cw4YNU8uWLbVw4UJVrlxZixcvLnT/L774Ql27dtWf//xnJSQk6PLLL9fgwYMvWE05U4VPRiTp6aef1u+//66kpCS99tpr2r59uzIyMvT3v/9dO3bsUHBwsFJSUpScnKx+/fppzZo12rt3r7744gvdf//9+vrrr+0+BQBAADr91F5vmqQCF1Lk5eUVOl5+fr42bdqklJQU97agoCClpKToyy+/LLRPly5dtGnTJnfy8eOPP2rVqlW64oorinyeFX6aRpIaNWqkLVu2aObMmZo4caL+9a9/yeFwqGXLlho3bpzuvPNOWZalVatW6f7779ewYcP0yy+/KDY2VpdccoliYmLsPgUAAErs7Isnpk6dqmnTphXY7+DBg3I6nQW+92JiYrRjx45Cj/3nP/9ZBw8eVLdu3WSM0e+//67bb79d//d//1fk+CxjjCny3vBaTk6OoqKilDj0IQWHhtkdTrE4Buy3O4QSiejzo90hAKhAfjcnlaZ3dOTIEUVGRpbJGKe/K8asu1qOiJALdziHvNyTmtf1XWVlZXnEeq71jP/+979Vt25dffHFF0pOTnZvHz9+vD755BN99dVXBfqkpaXphhtu0IMPPqjOnTtr165dGj16tIYPH67JkycXKc6AqIwAAOCPzpxqKWl/SYqMjCxS4lSzZk0FBwdr/37PPz7379+v2NjYQvtMnjxZN910k2677TZJUps2bXT06FH99a9/1f3336+goAuvCAmINSMAAODCQkND1bFjR4/7Z7lcLq1du9ajUnKmY8eOFUg4goODJUlFnXyhMgIAgI9yKUguL+oGJek7duxYDR06VElJSerUqZPmzZuno0ePatiwYZKkIUOGqG7dukpNTZUk9e3bV3PnzlX79u3d0zSTJ09W37593UnJhZCMAADgo5zGktOLaZqS9B00aJB++eUXTZkyRdnZ2WrXrp1Wr17tXtSamZnpUQmZNGmSLMvSpEmT9PPPP6tWrVrq27evHnrooSKPyQLWcsYC1vLHAlYApak8F7De8dm1Xi9gXdB9RZnGWhqojAAA4KNKawGrryMZAQDARxkvn7xr/ORBeSQjAAD4KKcsOUv4sLvT/f2Bf6RMAACgwqIyAgCAj3IZ79Z9uPzkEhWSEQAAfJTLyzUj3vQtT/4RJQAAqLCojAAA4KNcsuTyYhGqN33LE8kIAAA+yo47sNqBaRoAAGArKiM2id55QpX87Lf/nzdi7A6hRPbPrm13CCXWaNx6u0MAYKNAWcDqZ1+HAAAEDpe8vB28n6wZ8Y+UCQAAVFhURgAA8FHGy6tpjJ9URkhGAADwUTy1FwAA2CpQFrD6R5QAAKDCojICAICPYpoGAADYKlBuB880DQAAsBWVEQAAfBTTNAAAwFaBkowwTQMAAGxFZQQAAB8VKJURkhEAAHxUoCQjTNMAAABbURkBAMBHGXl3rxBTeqGUKZIRAAB8VKBM05CMAADgowIlGWHNiKSePXtqzJgxdocBAEBAojIiacWKFQoJCbE7DAAAPARKZYRkRFL16tXtDgEAgAICJRlhmkae0zSWZentt9/2eD86OlpLly6VJOXn52vkyJGKi4tTWFiYGjRooNTU1PINGACACoTKSDE9/vjjevfdd/X666+rfv36ysrKUlZW1jn3z8vLU15envt1Tk5OeYQJAKgAjLFkvKhueNO3PJGMFFNmZqaaNGmibt26ybIsNWjQ4Lz7p6amavr06eUUHQCgInHJ8uo+I970LU9M0xTTzTffrPT0dDVr1kyjRo3SmjVrzrv/xIkTdeTIEXc7XxUFAIBARDJyFsuyZIznPetOnjzp/rlDhw7as2ePHnjgAR0/flwDBw7UgAEDznk8h8OhyMhIjwYAQFGcXsDqTfMHTNOcpVatWtq3b5/79c6dO3Xs2DGPfSIjIzVo0CANGjRIAwYMUJ8+fXTo0CGuygEAlCrWjASoyy67TE8++aSSk5PldDo1YcIEj3uQzJ07V3FxcWrfvr2CgoK0fPlyxcbGKjo62r6gAQDwYyQjZ5kzZ46GDRum7t27q06dOpo/f742bdrkfr9q1aqaNWuWdu7cqeDgYF100UVatWqVgoKY8QIAlK5Auc8IyYiktLQ098916tTRBx984PH+4cOH3T8PHz5cw4cPL6fIAACBjGkaAABgK+NlZcRfkhHmFgAAgK2ojAAA4KOMpLPuNlHs/v6AZAQAAB/lkiWLO7ACAACULSojAAD4KK6mAQAAtnIZS1YA3GeEaRoAAGArKiMAAPgoY7y8msZPLqchGQEAwEcFypoRpmkAAICtqIwAAOCjAqUyQjICAICPCpSraUhGAADwUYGygJU1IwAAwFZURgAA8FGnKiPerBkpxWDKEMmITSql71QlK9TuMIql9vdhdodQIjVzj9odQondu3ub3SGUyKxGbewOAagQAmUBK9M0AADAVlRGAADwUea/zZv+/oBkBAAAH8U0DQAAQDmgMgIAgK8KkHkaKiMAAPiq/07TlLSphNM0Tz31lBISEhQWFqbOnTtrw4YN593/8OHDGjFihOLi4uRwONS0aVOtWrWqyONRGQEAwEfZcQfW1157TWPHjtXChQvVuXNnzZs3T71791ZGRoZq165dYP/8/Hz16tVLtWvX1htvvKG6devqp59+UnR0dJHHJBkBAABuc+fO1fDhwzVs2DBJ0sKFC7Vy5UotXrxY9913X4H9Fy9erEOHDumLL75QSEiIJCkhIaFYYzJNAwCAj/JmiubMK3FycnI8Wl5eXqHj5efna9OmTUpJSXFvCwoKUkpKir788stC+7z77rtKTk7WiBEjFBMTo9atW2vmzJlyOp1FPk+SEQAAfNXpdR/eNEnx8fGKiopyt9TU1EKHO3jwoJxOp2JiYjy2x8TEKDs7u9A+P/74o9544w05nU6tWrVKkydP1pw5c/Tggw8W+TSZpgEAoILLyspSZGSk+7XD4Si1Y7tcLtWuXVvPPvusgoOD1bFjR/3888969NFHNXXq1CIdg2QEAAAfVVoLWCMjIz2SkXOpWbOmgoODtX//fo/t+/fvV2xsbKF94uLiFBISouDgYPe2Fi1aKDs7W/n5+QoNvfBz2JimAQDAV5lSaMUQGhqqjh07au3ate5tLpdLa9euVXJycqF9unbtql27dsnlcrm3/fDDD4qLiytSIiKRjAAAgDOMHTtWixYt0gsvvKDt27frjjvu0NGjR91X1wwZMkQTJ05073/HHXfo0KFDGj16tH744QetXLlSM2fO1IgRI4o8JtM0AAD4KDueTTNo0CD98ssvmjJlirKzs9WuXTutXr3avag1MzNTQUH/q2XEx8frgw8+0N13363ExETVrVtXo0eP1oQJE4o8JskIAAC+zIZbuo8cOVIjR44s9L20tLQC25KTk7V+/foSj8c0DQAAsBWVEQAAfJQd0zR2oDJSRD179tSYMWPsDgMAEEjK+Woau1AZKaIVK1a477kPAED5sP7bvOnv+0hGiqh69ep2hwAAQIXENE0RnTlN8/TTT6tJkyYKCwtTTEyMBgwYcM5+eXl5BR5QBABAkTBNg8J8/fXXGjVqlF566SV16dJFhw4d0meffXbO/VNTUzV9+vRyjBAAUGF4m1CQjFRMmZmZqlKliq666ipVrVpVDRo0UPv27c+5/8SJEzV27Fj365ycHMXHx5dHqAAA+AWSkWLq1auXGjRooIYNG6pPnz7q06eP+vfvr8qVKxe6v8PhKNWnIwIAAoixTjVv+vsB1owUU9WqVbV582a9+uqriouL05QpU9S2bVsdPnzY7tAAABXM6af2etP8AclICVSqVEkpKSmaNWuWvvnmG+3du1f//Oc/7Q4LAAC/xDRNMb333nv68ccfdckll6hatWpatWqVXC6XmjVrZndoAICKhgWsKEx0dLRWrFihadOm6cSJE2rSpIleffVVtWrVyu7QAAAVTYCsGSEZKaIzn1JY2BMLAQBAyZCMAADgoyxzqnnT3x+QjAAA4KtYMwIAAGwVIGtGuLQXAADYisoIAAC+imkaAABgqwBJRpimAQAAtqIyAgCArwqQygjJCAAAvoqraQAAAMoelREAAHwUd2AFAAD2CpA1I0zTAAAAW5GMAAAAWzFNAwCAj7Lk5ZqRUoukbJGM2MR17Lhc1u92h1E8x47ZHUHJWP7yn2NBvzoj7A6hRIJbNbM7hBJxfpdhdwiAJy7tBQAAKHtURgAA8FUBcjUNyQgAAL4qQJIRpmkAAICtqIwAAOCjuAMrAACwF9M0AAAAZY/KCAAAvipAKiMkIwAA+KhAWTPCNA0AALAVlREAAHxVgNwOnmQEAABfxZoRAABgJ9aMAAAAlAMqIwAA+CqmaQAAgK28nKbxl2SEaRoAAGArKiMAAPgqpmkAAICtAiQZYZoGAADYisoIAAA+ivuMVEB5eXkaNWqUateurbCwMHXr1k0bN26UJKWlpcmyLK1du1ZJSUmqXLmyunTpooyMDI9jvPPOO+rQoYPCwsLUsGFDTZ8+Xb///rsdpwMAQIUQUMnI+PHj9eabb+qFF17Q5s2b1bhxY/Xu3VuHDh1y73P//fdrzpw5+vrrr1WpUiXdcsst7vc+++wzDRkyRKNHj9b333+vZ555RkuXLtVDDz10zjHz8vKUk5Pj0QAAwP8ETDJy9OhRLViwQI8++qj+9Kc/qWXLllq0aJHCw8P1/PPPu/d76KGH1KNHD7Vs2VL33XefvvjiC504cUKSNH36dN13330aOnSoGjZsqF69eumBBx7QM888c85xU1NTFRUV5W7x8fFlfq4AgArClELzAwGTjOzevVsnT55U165d3dtCQkLUqVMnbd++3b0tMTHR/XNcXJwk6cCBA5KkrVu3asaMGYqIiHC34cOHa9++fTp27Fih406cOFFHjhxxt6ysrLI4PQBABXR6zYg3zR+wgPUsISEh7p8t69Sjl10ulyQpNzdX06dP17XXXlugX1hYWKHHczgccjgcZRApACAg+ElC4Y2ASUYaNWqk0NBQrVu3Tg0aNJAknTx5Uhs3btSYMWOKdIwOHTooIyNDjRs3LsNIAQAILAGTjFSpUkV33HGH7r33XlWvXl3169fXrFmzdOzYMd16663aunXrBY8xZcoUXXXVVapfv74GDBigoKAgbd26Vd9++60efPDBcjgLAEBACZCbngVMMiJJDz/8sFwul2666Sb99ttvSkpK0gcffKBq1aoVqX/v3r313nvvacaMGXrkkUcUEhKi5s2b67bbbivjyAEAgShQ7jMSUMlIWFiYHn/8cT3++OMF3uvZs6eM8fxXa9euXYFtvXv3Vu/evcs0TgAAAklAJSMAAPgVpmkAAICdAmWaJmDuMwIAAHwTyQgAAL7KpjuwPvXUU0pISFBYWJg6d+6sDRs2FKnfsmXLZFmW+vXrV6zxSEYAAPBVNiQjr732msaOHaupU6dq8+bNatu2rXr37u2+G/m57N27V+PGjVP37t2LPSbJCAAAFdzZD2zNy8s7575z587V8OHDNWzYMLVs2VILFy5U5cqVtXjx4nP2cTqduvHGGzV9+nQ1bNiw2PGRjAAA4KNK69k08fHxHg9tTU1NLXS8/Px8bdq0SSkpKe5tQUFBSklJ0ZdffnnOOGfMmKHatWvr1ltvLdF5cjUNAAC+qpQu7c3KylJkZKR787memXbw4EE5nU7FxMR4bI+JidGOHTsK7fP555/r+eefV3p6eonDJBkBAMBXlVIyEhkZ6ZGMlJbffvtNN910kxYtWqSaNWuW+DgkIwAAQJJUs2ZNBQcHa//+/R7b9+/fr9jY2AL77969W3v37lXfvn3d204/6b5SpUrKyMhQo0aNLjgua0YAAPBRpbVmpKhCQ0PVsWNHrV271r3N5XJp7dq1Sk5OLrB/8+bNtW3bNqWnp7vb1VdfrUsvvVTp6emKj48v0rhURgAA8FU23A5+7NixGjp0qJKSktSpUyfNmzdPR48e1bBhwyRJQ4YMUd26dZWamqqwsDC1bt3ao390dLQkFdh+PiQjAADAbdCgQfrll180ZcoUZWdnq127dlq9erV7UWtmZqaCgkp3YoVkBAAAH2XXs2lGjhypkSNHFvpeWlraefsuXbq02OORjAAA4Kt4ai/KklWpkizLz379lp+udzYuuyMosclbrrE7hBKpX80/PytBFyfaHULJrP/G7ggAr/jZtyEAAAGEyggAALCT9d/mTX9/4J+1VAAAUGFQGQEAwFcxTQMAAOxk16W95Y1kBAAAXxUglRHWjAAAAFtRGQEAwJf5SXXDGyQjAAD4qEBZM8I0DQAAsBWVEQAAfFWALGAlGQEAwEcxTQMAAFAOqIwAAOCrmKYBAAB2YpoGAACgHFAZAQDAVzFNAwAAbEUyAgAA7MSaEVyQZVl6++237Q4DAAC/VuEqI06nU5ZlKSiIPAsA4OcCZJrG9m/snj17auTIkRo5cqSioqJUs2ZNTZ48Wcac+g3m5eVp3Lhxqlu3rqpUqaLOnTsrLS3N3X/p0qWKjo7Wu+++q5YtW8rhcCgzM1N5eXmaMGGC4uPj5XA41LhxYz3//PPuft9++63+9Kc/KSIiQjExMbrpppt08OBBj7hGjRql8ePHq3r16oqNjdW0adPc7yckJEiS+vfvL8uy3K8BACgtljFeN39gezIiSS+88IIqVaqkDRs2aP78+Zo7d66ee+45SdLIkSP15ZdfatmyZfrmm290/fXXq0+fPtq5c6e7/7Fjx/TII4/oueee03fffafatWtryJAhevXVV/X4449r+/bteuaZZxQRESFJOnz4sC677DK1b99eX3/9tVavXq39+/dr4MCBBeKqUqWKvvrqK82aNUszZszQhx9+KEnauHGjJGnJkiXat2+f+/XZ8vLylJOT49EAAMD/+MQ0TXx8vB577DFZlqVmzZpp27Zteuyxx9S7d28tWbJEmZmZqlOnjiRp3LhxWr16tZYsWaKZM2dKkk6ePKmnn35abdu2lST98MMPev311/Xhhx8qJSVFktSwYUP3eE8++aTat2/v7i9JixcvVnx8vH744Qc1bdpUkpSYmKipU6dKkpo0aaInn3xSa9euVa9evVSrVi1JUnR0tGJjY895bqmpqZo+fXpp/aoAAIGEaZryc/HFF8uyLPfr5ORk7dy5U9u2bZPT6VTTpk0VERHhbp988ol2797t3j80NFSJiYnu1+np6QoODlaPHj0KHW/r1q36+OOPPY7ZvHlzSfI47pnHlKS4uDgdOHCgWOc2ceJEHTlyxN2ysrKK1R8AELhOX03jTfMHPlEZOZfc3FwFBwdr06ZNCg4O9njv9JSLJIWHh3skM+Hh4Rc8bt++ffXII48UeC8uLs79c0hIiMd7lmXJ5XIV6xwcDoccDkex+gAAEEh8Ihn56quvPF6vX79eTZo0Ufv27eV0OnXgwAF17969yMdr06aNXC6XPvnkE/c0zZk6dOigN998UwkJCapUqeS/gpCQEDmdzhL3BwDgvJimKT+ZmZkaO3asMjIy9Oqrr+qJJ57Q6NGj1bRpU914440aMmSIVqxYoT179mjDhg1KTU3VypUrz3m8hIQEDR06VLfccovefvtt7dmzR2lpaXr99dclSSNGjNChQ4c0ePBgbdy4Ubt379YHH3ygYcOGFSu5SEhI0Nq1a5Wdna3//Oc/Xv8eAAA4U6BM0/hEMjJkyBAdP35cnTp10ogRIzR69Gj99a9/lXTqapUhQ4bonnvuUbNmzdSvXz9t3LhR9evXP+8xFyxYoAEDBujOO+9U8+bNNXz4cB09elSSVKdOHa1bt05Op1OXX3652rRpozFjxig6OrpY9yeZM2eOPvzwQ8XHx6t9+/Yl/wUAABDALGPsvQi5Z8+eateunebNm2dnGOUmJydHUVFRurTSdapkhVy4gy+xfCJ3LT5TvHU+vmTPyy3tDqFE6j/hn5+VoN/99LOy/hu7Iwgov5uTStM7OnLkiCIjI8tkjNPfFR1ueEjBoWElPo4z/4Q2L7u/TGMtDT6xZgQAABQUKM+mIRkBAMBXBcgCVtuTkTNv7Q4AAAKP7ckIAAA4N3+ZavEGyQgAAL7KmFPNm/5+wD+XvAMAgAqDyggAAD6Kq2kAAIC9AuRqGqZpAACAraiMAADgoyzXqeZNf39AMgIAgK9imgYAAKDsURkBAMBHcTUNAACwV4Dc9IxkBAAAH0VlBGUqqHK4gqxQu8MonhD//LiY/JN2h1BiEWur2B1CiVT67bDdIZTI8XoRdodQIo6LE+0OoeTWf2N3BPAB/vntAgBAIAiQq2lIRgAA8FGBMk3Dpb0AAMBWVEYAAPBVXE0DAADsxDQNAABAOaAyAgCAr+JqGgAAYCemaQAAAMoBlREAAHyVy5xq3vT3AyQjAAD4KtaMAAAAO1nycs1IqUVStlgzAgAAbEVlBAAAX8UdWAEAgJ24tBcAAASkp556SgkJCQoLC1Pnzp21YcOGc+67aNEide/eXdWqVVO1atWUkpJy3v0LQzICAICvMqXQium1117T2LFjNXXqVG3evFlt27ZV7969deDAgUL3T0tL0+DBg/Xxxx/ryy+/VHx8vC6//HL9/PPPRR6TZAQAAB9lGeN1k6ScnByPlpeXd84x586dq+HDh2vYsGFq2bKlFi5cqMqVK2vx4sWF7v/yyy/rzjvvVLt27dS8eXM999xzcrlcWrt2bZHPs1jJSM+ePTVmzJjidCmxvXv3yrIspaenl8nxly5dqujo6DI5NgAAviQ+Pl5RUVHulpqaWuh++fn52rRpk1JSUtzbgoKClJKSoi+//LJIYx07dkwnT55U9erVixxfsRawrlixQiEhIcXpUmLx8fHat2+fatas6fWxEhISNGbMGI9EatCgQbriiiu8PjYAAGXG9d/mTX9JWVlZioyMdG92OByF7n7w4EE5nU7FxMR4bI+JidGOHTuKNOSECRNUp04dj4TmQoqVjBQny/FWcHCwYmNjz/m+MUZOp1OVKpXsgqDw8HCFh4eXNDwAAMrcmVMtJe0vSZGRkR7JSFl5+OGHtWzZMqWlpSksLKzI/Uo8TZOQkKCZM2fqlltuUdWqVVW/fn09++yz7n3z8/M1cuRIxcXFKSwsTA0aNPAoC1mWpQULFuhPf/qTwsPD1bBhQ73xxhvu98+epklLS5NlWXr//ffVsWNHORwOff7559q9e7euueYaxcTEKCIiQhdddJE++ugjj5h/+ukn3X333bIsS5Z16n50hU3TLFiwQI0aNVJoaKiaNWuml156yeN9y7L03HPPqX///qpcubKaNGmid999tzi/QgAAfFbNmjUVHBys/fv3e2zfv3//eQsEkjR79mw9/PDDWrNmjRITE4s1rlcLWOfMmaOkpCRt2bJFd955p+644w5lZGRIkh5//HG9++67ev3115WRkaGXX35ZCQkJHv0nT56s6667Tlu3btWNN96oG264Qdu3bz/vmPfdd58efvhhbd++XYmJicrNzdUVV1yhtWvXasuWLerTp4/69u2rzMxMSaemlurVq6cZM2Zo37592rdvX6HHfeuttzR69Gjdc889+vbbb/W3v/1Nw4YN08cff+yx3/Tp0zVw4EB98803uuKKK3TjjTfq0KFD54w3Ly+vwMIhAACKpJyvpgkNDVXHjh09Fp+eXoyanJx8zn6zZs3SAw88oNWrVyspKal4g8rLZOSKK67QnXfeqcaNG2vChAmqWbOm+8s7MzNTTZo0Ubdu3dSgQQN169ZNgwcP9uh//fXX67bbblPTpk31wAMPKCkpSU888cR5x5wxY4Z69eqlRo0aqXr16mrbtq3+9re/qXXr1mrSpIkeeOABNWrUyF2xqF69uoKDg1W1alXFxsaeM7ObPXu2br75Zt15551q2rSpxo4dq2uvvVazZ8/22O/mm2/W4MGD1bhxY82cOVO5ubnnvZ46NTXVY9FQfHz8BX+vAABI+t8dWL1pxTR27FgtWrRIL7zwgrZv36477rhDR48e1bBhwyRJQ4YM0cSJE937P/LII5o8ebIWL16shIQEZWdnKzs7W7m5uUUe06tk5MwyjGVZio2NdV+HfPPNNys9PV3NmjXTqFGjtGbNmgL9z86ykpOTL1gZOTvjys3N1bhx49SiRQtFR0crIiJC27dvd1dGimr79u3q2rWrx7auXbsWiOfMc65SpYoiIyPPee21JE2cOFFHjhxxt6ysrGLFBQAIXKfvwOpNK65BgwZp9uzZmjJlitq1a6f09HStXr3avag1MzPTY5ZhwYIFys/P14ABAxQXF+duZ/8xfz5e3Q7+7CtrLMuSy3Vq6W6HDh20Z88evf/++/roo480cOBApaSkeKwLKYkqVap4vB43bpw+/PBDzZ49W40bN1Z4eLgGDBig/Px8r8Y5l/Odc2EcDsc5Vy0DAOCLRo4cqZEjRxb6XlpamsfrvXv3ej1emd70LDIyUoMGDdKiRYv02muv6c033/RYX7F+/XqP/devX68WLVoUa4x169bp5ptvVv/+/dWmTRvFxsYW+MWEhobK6XSe9zgtWrTQunXrChy7ZcuWxYoHAIBSY8M0jR3K7EF5c+fOVVxcnNq3b6+goCAtX75csbGxHlewLF++XElJSerWrZtefvllbdiwQc8//3yxxmnSpIlWrFihvn37yrIsTZ48uUClIiEhQZ9++qluuOEGORyOQu9dcu+992rgwIFq3769UlJS9I9//EMrVqzwuDIHAIDyZLlONW/6+4Myq4xUrVpVs2bNUlJSki666CLt3btXq1atUlDQ/4acPn26li1bpsTERL344ot69dVXi12JmDt3rqpVq6YuXbqob9++6t27tzp06OCxz4wZM7R37141atRItWrVKvQ4/fr10/z58zV79my1atVKzzzzjJYsWaKePXsW+9wBAEDRWcbYU8OxLEtvvfWW+vXrZ8fwtsnJyVFUVJT+GPkXVbJC7Q6neELKrJBWpkz+SbtDKLEDg1vbHUKJ1P7qsN0hlMjxehF2h1Aijl/P/ZwRn7f+G7sjKLbfzUml6R0dOXKkzG4kdvq7omen+1WpUtFvHna2338/obQND5VprKXBP79dAAAIBCV88q5Hfz/AU3sBAICtbKuM2DQ7BACA3yitZ9P4OqZpAADwVd5enusnyQjTNAAAwFZURgAA8FVGkjf3CvGPwgjJCAAAvoo1IwAAwF5GXq4ZKbVIyhRrRgAAgK2ojAAA4KsC5GoakhEAAHyVS5LlZX8/wDQNAACwFZURAAB8FFfTAAAAewXImhGmaQAAgK2ojAAA4KsCpDJCMmITk58v480KaTvk59sdQYm48k/aHUKJhf3HT5bCn8Xa87PdIZRIeIZ/fsb9+SnoO19qb3cIxeY6dkL66zvlM1iAJCNM0wAAAFtRGQEAwFcFyH1GSEYAAPBRXNoLAADsxZoRAACAskdlBAAAX+UykuVFdcPlH5URkhEAAHwV0zQAAABlj8oIAAA+y8vKiPyjMkIyAgCAr2KaBgAAoOxRGQEAwFe5jLyaauFqGgAA4BXjOtW86e8HmKYBAAC2ojICAICvCpAFrCQjAAD4KtaMAAAAWwVIZYQ1I15aunSpoqOj7Q4DAAC/RWUEAABfZeRlZaTUIilTJCMAAPgqpmkqntWrV6tbt26Kjo5WjRo1dNVVV2n37t2SpL1798qyLK1YsUKXXnqpKleurLZt2+rLL7/0OMbSpUtVv359Va5cWf3799evv/5qx6kAAFBhBFQycvToUY0dO1Zff/211q5dq6CgIPXv318u1/9uCnP//fdr3LhxSk9PV9OmTTV48GD9/vvvkqSvvvpKt956q0aOHKn09HRdeumlevDBB887Zl5ennJycjwaAABF4nJ53/xAQE3TXHfddR6vFy9erFq1aun7779XRESEJGncuHG68sorJUnTp09Xq1attGvXLjVv3lzz589Xnz59NH78eElS06ZN9cUXX2j16tXnHDM1NVXTp08vozMCAFRoTNNUPDt37tTgwYPVsGFDRUZGKiEhQZKUmZnp3icxMdH9c1xcnCTpwIEDkqTt27erc+fOHsdMTk4+75gTJ07UkSNH3C0rK6s0TgUAgAojoCojffv2VYMGDbRo0SLVqVNHLpdLrVu3Vn5+vnufkJAQ98+WZUmSxzROcTkcDjkcjpIHDQAIXAFSGQmYZOTXX39VRkaGFi1apO7du0uSPv/882Ido0WLFvrqq688tq1fv77UYgQAwAN3YK1YqlWrpho1aujZZ59VXFycMjMzdd999xXrGKNGjVLXrl01e/ZsXXPNNfrggw/Ou14EAABcWMCsGQkKCtKyZcu0adMmtW7dWnfffbceffTRYh3j4osv1qJFizR//ny1bdtWa9as0aRJk8ooYgBAoDPG5XXzBwFTGZGklJQUff/99x7bzBnzaeasubXo6OgC22655RbdcsstHtvuueeeUo4UAACdWvPhzVQLa0YAAIBXjJdrRvwkGQmYaRoAAOCbqIwAAOCrXC7J8mLdB2tGAACAV5imAQAAKHtURgAA8FHG5ZLxYpqGS3sBAIB3mKYBAAAoe1RGAADwVS4jWRW/MkIyAgCArzJGkjeX9vpHMsI0DQAAsBWVEQAAfJRxGRkvpmnOfr6aryIZAQDAVxmXvJum8Y9Le5mmAQDARxmX8bqVxFNPPaWEhASFhYWpc+fO2rBhw3n3X758uZo3b66wsDC1adNGq1atKtZ4JCMAAMDttdde09ixYzV16lRt3rxZbdu2Ve/evXXgwIFC9//iiy80ePBg3XrrrdqyZYv69eunfv366dtvvy3ymJbxlwmlCuLIkSOKjo7WJY7+qmSF2B1OQHDl/253CCWWe017u0Mokai1GXaHUCImP9/uEErEn/8vvvuJZnaHUGyu43n61+hZOnz4sKKiospkjJycHEVFRambrlAllfy74ned1OdapaysLEVGRrq3OxwOORyOQvt07txZF110kZ588klJksvlUnx8vO666y7dd999BfYfNGiQjh49qvfee8+97eKLL1a7du20cOHCogVqUK6ysrJO306PRqPRaH7csrKyyuy74vjx4yY2NrZU4oyIiCiwberUqYWOm5eXZ4KDg81bb73lsX3IkCHm6quvLrRPfHy8eeyxxzy2TZkyxSQmJhb5fFnAWs7q1KmjrKwsVa1aVZZlleqxc3JyFB8fXyAD9gf+Gjtxly/iLn/+GntZxm2M0W+//aY6deqU6nHPFBYWpj179ii/FKp1xpgC3zfnqoocPHhQTqdTMTExHttjYmK0Y8eOQvtkZ2cXun92dnaRYyQZKWdBQUGqV69emY4RGRnpV//TOJO/xk7c5Yu4y5+/xl5WcZfV9MyZwsLCFBYWVubj+AIWsAIAAElSzZo1FRwcrP3793ts379/v2JjYwvtExsbW6z9C0MyAgAAJEmhoaHq2LGj1q5d697mcrm0du1aJScnF9onOTnZY39J+vDDD8+5f2GYpqlAHA6Hpk6des65QF/mr7ETd/ki7vLnr7H7a9y+YOzYsRo6dKiSkpLUqVMnzZs3T0ePHtWwYcMkSUOGDFHdunWVmpoqSRo9erR69OihOXPm6Morr9SyZcv09ddf69lnny3ymFzaCwAAPDz55JN69NFHlZ2drXbt2unxxx9X586dJUk9e/ZUQkKCli5d6t5/+fLlmjRpkvbu3asmTZpo1qxZuuKKK4o8HskIAACwFWtGAACArUhGAACArUhGAACArUhGgPPo2bOnxowZY3cYpaIinUtFV57/Vnv37pVlWUpPTy+T4y9dulTR0dFlcuzS4OvxBQoWsKLMZGdnKzU1VStXrtS//vUvRUVFqXHjxvrLX/6ioUOHqnLlynaHeEGHDh1SSEiIqlatancoXqtI51LRFeffyrIsvfXWW+rXr1+JxnI6nfrll19Us2ZNVark3d0eEhISNGbMGI9E6vjx4/rtt99Uu3Ztr45dVpYuXaoxY8bo8OHDdocS0LjPCMrEjz/+qK5duyo6OlozZ85UmzZt5HA4tG3bNj377LOqW7eurr76arvDvKDq1avbHUKpqUjn4uucTqcsy1JQUMmKz+X5bxUcHHzeO2UaY+R0OkucqISHhys8PLyk4SFQFPmRekAx9O7d29SrV8/k5uYW+r7L5TLGGPOf//zH3HrrraZmzZqmatWq5tJLLzXp6enu/aZOnWratm1rXnzxRdOgQQMTGRlpBg0aZHJycsrlPHr06GFGjx5tjDFGUoEnWUZFRZklS5YYY0497XLEiBEmNjbWOBwOU79+fTNz5sxyibMozjyXp556yjRu3Ng4HA5Tu3Ztc91119ka24kTJ8xdd91latWqZRwOh+natavZsGGDMcaYjz/+2EgyH330kenYsaMJDw83ycnJZseOHR7HePvtt0379u2Nw+Ewf/jDH8y0adPMyZMnizR+jx49zIgRI8yIESNMZGSkqVGjhpk0aZL7c3rixAlzzz33mDp16pjKlSubTp06mY8//tjdf8mSJSYqKsq88847pkWLFiY4ONjs2bPHnDhxwowfP97Uq1fPhIaGmkaNGpnnnnvO3W/btm2mT58+pkqVKqZ27drmL3/5i/nll1/c/1Y9evQwVatWNd27dzehoaHGsiwTGRlpnnnmGWOMMQ0aNCjwNNazP3eSzNNPP2369OljwsLCzB/+8AezfPly9/t79uwxksyWLVs8ft+rVq0yHTp0MCEhIebjjz82u3btMldffbWpXbu2qVKliklKSjIffvihx+/w7FjO/N2c6emnnzYNGzY0ISEhpmnTpubFF1/0eF+SWbRokenXr58JDw83jRs3Nu+88455//33TdeuXU1UVJSpXr26ufLKK82uXbs8zuPNN980PXv2NOHh4SYxMdF88cUXHsdesmSJiY+PN+Hh4aZfv35m9uzZBeJD+SMZQak7ePCgsSzLpKamXnDflJQU07dvX7Nx40bzww8/mHvuucfUqFHD/Prrr8aYU8lIRESEufbaa822bdvMp59+amJjY83//d//lfVpGGOKl4w8+uijJj4+3nz66adm79695rPPPjOvvPJKucRZFKfPZePGjSY4ONi88sorZu/evWbz5s1m/vz5tsY2atQoU6dOHbNq1Srz3XffmaFDh5pq1aqZX3/91f3l2LlzZ5OWlma+++470717d9OlSxd3/08//dRERkaapUuXmt27d5s1a9aYhIQEM23atCKN36NHDxMREWFGjx5tduzYYf7+97+bypUrm2effdYYY8xtt91munTpYj799FOza9cu8+ijjxqHw2F++OEHY8ypL7iQkBDTpUsXs27dOrNjxw5z9OhRM3DgQBMfH29WrFhhdu/ebT766COzbNkyY8ypRLxWrVpm4sSJZvv27Wbz5s2mV69e5tJLL/VIRizLMuHh4Wbq1Klm1qxZRpIJCgoyO3bsMAcOHDCSTPXq1c1bb71lNm3aVOBzJ8nUqFHDLFq0yGRkZJhJkyaZ4OBg8/333xtjzp2MJCYmmjVr1phdu3aZX3/91aSnp5uFCxeabdu2mR9++MFMmjTJhIWFmZ9++skYY8yvv/5q6tWrZ2bMmGH27dtn9u3b5/7dnPllv2LFChMSEmKeeuopk5GRYebMmWOCg4PNP//5T4+Y69WrZ1555RWzc+dOM2rUKBMREWGWLFli3nzzTbNz506zZcsW07dvX9OmTRvjdDrd59G8eXPz3nvvmYyMDDNgwADToEEDd1K6fv16ExQUZB555BGTkZFh5s+fb6Kjo0lGfADJCErd+vXrjSSzYsUKj+01atQwVapUMVWqVDHjx483n332mYmMjDQnTpzw2K9Ro0buv/ymTp1qKleu7FEJuffee03nzp3L/kRM8ZKRu+66y1x22WXuv6Z9zelzefPNN01kZGS5VZcuJDc314SEhJiXX37ZvS0/P9/UqVPHzJo1y6MyctrKlSuNJHP8+HFjjDF//OMfC1ShXnrpJRMXF1ekGHr06GFatGjh8W83YcIE06JFC/PTTz+Z4OBg8/PPP3v0+eMf/2gmTpxojDn1hSvJo6qXkZFhJHlUD870wAMPmMsvv9xjW1ZWlpFkOnXq5E5GHA6H+ctf/uLeJykpyVSuXNksWLDAGHPqc9mmTZtzfu4kmdtvv91jW+fOnc0dd9xhjDl3MvL222+f8/d1WqtWrcwTTzzhft2gQQPz2GOPeexzdjLSpUsXM3z4cI99rr/+enPFFVd4xDxp0iT369zcXCPJvP/++x79fvnlFyPJbNu2zX0eZ1aevvvuOyPJbN++3RhjzODBgz3GMcaYQYMGkYz4AK6mQbnZsGGD0tPT1apVK+Xl5Wnr1q3Kzc1VjRo1FBER4W579uzR7t273f0SEhI8FvLFxcXpwIEDdpzCed18881KT09Xs2bNNGrUKK1Zs8bukArVq1cvNWjQQA0bNtRNN92kl19+WceOHbMtnt27d+vkyZPq2rWre1tISIg6deqk7du3u7clJia6f46Li5Mk9+dg69atmjFjhsfnaPjw4dq3b1+Rz+3iiy+WZVnu18nJydq5c6e2bdsmp9Oppk2behz/k08+8fichoaGesSYnp6u4OBg9ejRo9Dxtm7dqo8//tjjmM2bN5d0atHnuY5bp04dORwOj/8G9uzZc97P3dkPLEtOTvb43RYmKSnJ43Vubq7GjRunFi1aKDo6WhEREdq+fbsyMzPPe5yzbd++3ePfWpK6du1aIJ4zz7lKlSqKjIzUN998o8GDB6thw4aKjIxUQkKCJHnEcL7Pyfbt2923ND+tOA9zQ9lhAStKXePGjWVZljIyMjy2N2zYUJLci9lyc3MVFxentLS0Asc481K7kJAQj/csy5LL5SrdoIvAsiyZsy4+O3nypPvnDh06aM+ePXr//ff10UcfaeDAgUpJSdEbb7xR3qGeV9WqVbV582alpaVpzZo1mjJliqZNm6aNGzf69CWOZ34OTicNpz8Hubm5mj59uq699toC/cLCwrwaNzc3V8HBwdq0aZOCg4M93ouIiHD/HB4e7pHMXGjRZm5urvr27atHHnmkwHunH0h22vnOXZKeeeYZBQcHl+rnrkqVKh6vx40bpw8//FCzZ89W48aNFR4ergEDBig/P9+rcc6lsP/uH3vsMSUmJmrRokWqU6eOXC6XWrdu7RHDhX5X8E0kIyh1NWrUUK9evfTkk0/qrrvuKvA/tdM6dOig7OxsVapUyf0Xji+rVauW9u3b5369c+fOAn91R0ZGatCgQRo0aJAGDBigPn366NChQz53JUulSpWUkpKilJQUTZ06VdHR0frnP/9Z6Jd5WWvUqJFCQ0O1bt06NWjQQNKpJG/jxo1FvtdGhw4dlJGRocaNG5c4jq+++srj9fr169WkSRO1b99eTqdTBw4cUPfu3Yt8vDZt2sjlcumTTz5RSkpKoTG/+eabSkhIKHClytlJz/mEhITI4XDouuuuO+fnbv369RoyZIjHubVv377IY0jSunXrdPPNN6t///6STiVTe/fu9dgnNDRUTqfzvMdp0aKF1q1bp6FDh3ocu2XLluft53K5lJ2drddff9397/D5558X6xxatGhR6L8z7EcygjLx9NNPq2vXrkpKStK0adOUmJiooKAgbdy4UTt27FDHjh2VkpKi5ORk9evXT7NmzVLTpk3173//WytXrlT//v0LlIntdtlll+nJJ59UcnKynE6nJkyY4PFX2Ny5cxUXF6f27dsrKChIy5cvV2xsrM9VG9577z39+OOPuuSSS1StWjWtWrVKLpdLzZo1syWeKlWq6I477tC9996r6tWrq379+po1a5aOHTumW2+9VVu3br3gMaZMmaKrrrpK9evX14ABAxQUFKStW7fq22+/1YMPPlikODIzMzV27Fj97W9/0+bNm/XEE09ozpw5atq0qW688UYNGTJEc+bMUfv27fXLL79o7dq1SkxM1JVXXlno8RISEjR06FDdcsstevzxx9W2bVv99NNPOnDggAYOHKgRI0Zo0aJFGjx4sMaPH6/q1atr165dWrZsWYEK3PlERUVp4cKFql27tsLCwgr93C1fvlxJSUnq1q2bXn75ZW3YsEHPP/98kceQpCZNmmjFihXq27evLMvS5MmTC1QcEhIS9Omnn+qGG26Qw+FQzZo1Cxzn3nvv1cCBA9W+fXulpKToH//4h1asWKGPPvrovONblqWIiAg9++yziouLU2Zmpu67775incOoUaPUtWtXzZ49W9dcc40++OADrV69uljHQNlgzQjKRKNGjbRlyxalpKRo4sSJatu2rZKSkvTEE09o3LhxeuCBB2RZllatWqVLLrlEw4YNU9OmTXXDDTfop59+UkxMjN2nUMCcOXMUHx+v7t27689//rPGjRvnceO2qlWratasWUpKStJFF12kvXv3atWqVSW+10RZiY6O1ooVK3TZZZepRYsWWrhwoV599VW1atXKtpgefvhhXXfddbrpppvUoUMH7dq1Sx988IGqVatWpP69e/fWe++9pzVr1uiiiy7SxRdfrMcee8xdaSmKIUOG6Pjx4+rUqZNGjBih0aNH669//askacmSJRoyZIjuueceNWvWTP369dPGjRtVv3798x5zwYIFGjBggO688041b95cw4cP19GjRyWdWvuxbt06OZ1OXX755WrTpo3GjBmj6Ohoj+meCxk0aJA+/fRTXXLJJercuXOhn7vp06dr2bJlSkxM1IsvvqhXX331gpWIs82dO1fVqlVTly5d1LdvX/Xu3VsdOnTw2GfGjBnau3evGjVqpFq1ahV6nH79+mn+/PmaPXu2WrVqpWeeeUZLlixRz549zzu+ZVm6/fbbtWnTJrVu3Vp33323Hn300WKdw8UXX6xFixZp/vz5atu2rdasWaNJkyYV6xgoG9yBFUDA69mzp9q1a6d58+bZHUqp8/YOrUB58K0/2QAAQMAhGQEAALZimgYAANiKyggAALAVyQgAALAVyQgAALAVyQgAALAVyQgAALAVyQgAALAVyQgAALAVyQgAALDV/wNzVbnGT1nZWwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1).to(torch.bfloat16)\n",
    "display_qk_heatmap(qk_per_token_after_masking_after_softmax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## values (almost the end of attention)\n",
    "these scores (0-1) are used to determine how much of value matrix is used per token\n",
    "<br>\n",
    "&gt; just like keys, value weights are also shared acorss every 4 attention heads (to save computation)\n",
    "<br>\n",
    "&gt; as a result, the shape of the value weight matrix below is [20x128x2560]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 128, 4096])\n",
      "torch.Size([2, 128])\n"
     ]
    }
   ],
   "source": [
    "#v_layer0 = model[\"model.layers.0.self_attn.v_proj.weight\"]\n",
    "#v_layer0 = v_layer0.view(n_kv_heads, v_layer0.shape[0] // n_kv_heads, dim)\n",
    "v_layer0 = v_layer0.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "print(v_layer0.shape)\n",
    "\n",
    "#v_layer0_bias = model[\"model.layers.0.self_attn.v_proj.bias\"]\n",
    "v_layer0_bias = v_layer0_bias.reshape(n_kv_heads, -1)\n",
    "print(v_layer0_bias.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the first layer, first head value weight matrix is given below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128, 4096])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_layer0_head0 = v_layer0[0]\n",
    "v_layer0_head0.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([128])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_layer0_bias_head0 = v_layer0_bias[0]\n",
    "v_layer0_bias_head0.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## value vectors\n",
    "we now use the value weghts to get the attention values per token, this is of size [7x128] where 7 is the number of tokens in the prompt and 128 is the dim of the value vector per token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_per_token = torch.matmul(token_embeddings, v_layer0_head0.T.float()) + v_layer0_bias_head0.float()\n",
    "v_per_token.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## attention\n",
    "the resultant attention vector after multipying with the values per token is of shape [7*128]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax.float(), v_per_token.float())\n",
    "qkv_attention.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# multi head attention\n",
    "WE NOW HAVE THE ATTENTION VALUE OF THE FIRST LAYER AND FIRST HEAD\n",
    "<br>\n",
    "now im going to run a loop and perform the exact same math as the cells above but for every head in the first layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 128])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qkv_attention_store = []\n",
    "GQA_num = n_heads // n_kv_heads # \n",
    "for head in range(n_heads):\n",
    "    q_layer0_head = q_layer0[head]\n",
    "    k_layer0_head = k_layer0[head//GQA_num] \n",
    "    v_layer0_head = v_layer0[head//GQA_num]\n",
    "    q_layer0_bias_head = q_layer0_bias[head]\n",
    "    k_layer0_bias_head = k_layer0_bias[head//GQA_num]\n",
    "    v_layer0_bias_head = v_layer0_bias[head//GQA_num]\n",
    "    q_per_token = torch.matmul(token_embeddings, q_layer0_head.T.float()) + q_layer0_bias_head.float()\n",
    "    k_per_token = torch.matmul(token_embeddings, k_layer0_head.T.float()) + k_layer0_bias_head.float()\n",
    "    v_per_token = torch.matmul(token_embeddings, v_layer0_head.T.float()) + v_layer0_bias_head.float()\n",
    "\n",
    "    q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)\n",
    "    q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)\n",
    "    q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers * freqs_cis[:len(tokens)])\n",
    "    q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)\n",
    "\n",
    "    k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)\n",
    "    k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)\n",
    "    k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis[:len(tokens)])\n",
    "    k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)\n",
    "\n",
    "    qk_per_token = torch.matmul(q_per_token_rotated, k_per_token_rotated.T)/(128)**0.5\n",
    "    mask = torch.full((len(tokens), len(tokens)), float(\"-inf\"), device=tokens.device)\n",
    "    mask = torch.triu(mask, diagonal=1)\n",
    "    qk_per_token_after_masking = qk_per_token + mask\n",
    "    qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1) # .to(torch.bfloat16)\n",
    "    qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)\n",
    "    qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)\n",
    "    qkv_attention_store.append(qkv_attention)\n",
    "\n",
    "print(qkv_attention_store[0].shape)\n",
    "len(qkv_attention_store)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we now have a the qkv_attention matrix for all 20 heads on the first layer, next im going to merge all attention scores into one large matrix of size [7x2560]\n",
    "<br>\n",
    "we are almost at the end :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stacked_qkv_attention = torch.cat(qkv_attention_store, dim=-1)\n",
    "stacked_qkv_attention.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# weight matrix, one of the final steps\n",
    "one of the last things to do for a layer 0 attention is, is to multiply the weight matrix of the "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4096, 4096])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_layer0 = model[\"transformer.encoder.layers.0.self_attention.dense.weight\"]\n",
    "w_layer0.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### this is a simple linear layer, so we just matmul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_delta = torch.matmul(stacked_qkv_attention, w_layer0.T.float())\n",
    "embedding_delta.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we now have the change in the embedding value after attention, that should be adding to the original token embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_after_edit = token_embeddings_unnormalized + embedding_delta\n",
    "embedding_after_edit.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## we normalize and then run a feed forward neural network through the embedding delta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_after_edit_normalized = rms_norm(embedding_after_edit, model[\"transformer.encoder.layers.0.post_attention_layernorm.weight\"])\n",
    "embedding_after_edit_normalized.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## loading the ff weights and implementing the feed forward network\n",
    "in llama3, they used a SwiGLU feedforward network, this network architecture is really good at adding non linearity when needed by the model.\n",
    "<br>\n",
    "its pretty standard to use this feed forward network architecture in llms these days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1 = model[\"transformer.encoder.layers.0.mlp.dense_h_to_4h.weight\"]\n",
    "w2 = model[\"transformer.encoder.layers.0.mlp.dense_4h_to_h.weight\"]\n",
    "\n",
    "x = torch.matmul(embedding_after_edit_normalized, w1.T.float())\n",
    "x = torch.chunk(x, 2, dim=-1)\n",
    "output_after_feedforward = torch.matmul(torch.functional.F.silu(x[0]) * x[1], w2.T.float())\n",
    "output_after_feedforward.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WE FINALLY HAVE NEW EDITED EMBEDDINGS FOR EACH TOKEN AFTER THE FIRST LAYER\n",
    "just 39 more layers to go before we are done (one for loop away)\n",
    "<br>\n",
    "you can imagine this edited embedding as having information about all queries asked on the first layer\n",
    "<br>\n",
    "now each layer will encode more and more complex queries on the quesions asked, until we have an embedding that knows everything about the next token that we need."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_0_embedding = embedding_after_edit+output_after_feedforward\n",
    "layer_0_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# god, everything all at once\n",
    "yep, this is it. everything we did before, all at once, for every single layer.\n",
    "<br>\n",
    "\n",
    "# have fun reading :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_cache, v_cache = [], []   # init k v cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_embedding = token_embeddings_unnormalized\n",
    "GQA_num = n_heads // n_kv_heads\n",
    "for layer in range(n_layers):\n",
    "    k_cache.append([])\n",
    "    v_cache.append([])\n",
    "    qkv_attention_store = []\n",
    "    layer_embedding_norm = rms_norm(final_embedding, model[f\"transformer.encoder.layers.{layer}.input_layernorm.weight\"])\n",
    "    q_k_v_layer = model[f\"transformer.encoder.layers.{layer}.self_attention.query_key_value.weight\"]\n",
    "    head_dim = q_k_v_layer.shape[1] // n_heads\n",
    "    q_layer, k_layer, v_layer = q_k_v_layer.split([ \n",
    "                                                    n_heads * n_kv_channels,\n",
    "                                                    n_kv_heads * n_kv_channels,\n",
    "                                                    n_kv_heads * n_kv_channels,\n",
    "                                                  ], dim=0)\n",
    "    q_layer = q_layer.reshape(n_heads, n_kv_channels, -1)\n",
    "\n",
    "    q_k_v_layer_bias = model[f\"transformer.encoder.layers.{layer}.self_attention.query_key_value.bias\"]\n",
    "    q_layer_bias, k_layer_bias, v_layer_bias = q_k_v_layer_bias.split([\n",
    "                                                                        n_heads * n_kv_channels,\n",
    "                                                                        n_kv_heads * n_kv_channels,\n",
    "                                                                        n_kv_heads * n_kv_channels,\n",
    "                                                                      ])\n",
    "    q_layer_bias = q_layer_bias.reshape(n_heads, -1)\n",
    "    \n",
    "    k_layer = k_layer.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "    k_layer_bias = k_layer_bias.reshape(n_kv_heads, -1)\n",
    "    \n",
    "    v_layer = v_layer.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "    v_layer_bias = v_layer_bias.reshape(n_kv_heads, -1)\n",
    "\n",
    "    for head in range(n_heads):\n",
    "        q_layer_head = q_layer[head]\n",
    "        k_layer_head = k_layer[head//GQA_num]\n",
    "        v_layer_head = v_layer[head//GQA_num]\n",
    "        q_layer_bias_head = q_layer_bias[head]\n",
    "        k_layer_bias_head = k_layer_bias[head//GQA_num]\n",
    "        v_layer_bias_head = v_layer_bias[head//GQA_num]\n",
    "        q_per_token = torch.matmul(layer_embedding_norm, q_layer_head.T.float()) + q_layer_bias_head.float()\n",
    "        k_per_token = torch.matmul(layer_embedding_norm, k_layer_head.T.float()) + k_layer_bias_head.float()\n",
    "        v_per_token = torch.matmul(layer_embedding_norm, v_layer_head.T.float()) + v_layer_bias_head.float()\n",
    "        if head % GQA_num == 0:   # qwen1.5-4B use MHA，GQA_num=1\n",
    "            v_cache[-1].append(v_per_token)   # cache v vector\n",
    "        q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)\n",
    "        q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)\n",
    "        q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers * freqs_cis)\n",
    "        q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)\n",
    "        k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)\n",
    "        k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)\n",
    "        k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis)\n",
    "        k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)\n",
    "        if head % GQA_num == 0:   # \n",
    "            k_cache[-1].append(k_per_token)  # cache k vector\n",
    "        qk_per_token = torch.matmul(q_per_token_rotated, k_per_token_rotated.T)/(128)**0.5\n",
    "        mask = torch.full((len(token_embeddings_unnormalized), len(token_embeddings_unnormalized)), float(\"-inf\"))\n",
    "        mask = torch.triu(mask, diagonal=1)\n",
    "        qk_per_token_after_masking = qk_per_token + mask\n",
    "        qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1) #.to(torch.bfloat16)\n",
    "        qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_per_token)\n",
    "        qkv_attention_store.append(qkv_attention)\n",
    "\n",
    "    stacked_qkv_attention = torch.cat(qkv_attention_store, dim=-1)\n",
    "    w_layer = model[f\"transformer.encoder.layers.{layer}.self_attention.dense.weight\"]\n",
    "    embedding_delta = torch.matmul(stacked_qkv_attention, w_layer.T.float())\n",
    "    embedding_after_edit = final_embedding + embedding_delta\n",
    "    embedding_after_edit_normalized = rms_norm(embedding_after_edit, model[f\"transformer.encoder.layers.{layer}.post_attention_layernorm.weight\"])\n",
    "    w1 = model[f\"transformer.encoder.layers.{layer}.mlp.dense_h_to_4h.weight\"]\n",
    "    w2 = model[f\"transformer.encoder.layers.{layer}.mlp.dense_4h_to_h.weight\"]\n",
    "    x = torch.matmul(embedding_after_edit_normalized, w1.T.float())\n",
    "    x = torch.chunk(x, 2, dim=-1)\n",
    "    output_after_feedforward = torch.matmul(torch.functional.F.silu(x[0]) * x[1], w2.T.float())\n",
    "    final_embedding = embedding_after_edit+output_after_feedforward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([8, 128])\n",
      "torch.Size([8, 128])\n",
      "28\n",
      "2\n",
      "torch.Size([8, 128])\n",
      "torch.Size([8, 128])\n",
      "28\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "# k v cache info\n",
    "print(k_cache[0][0].shape)\n",
    "print(k_cache[0][-1].shape)\n",
    "print(len(k_cache))\n",
    "print(len(k_cache[0]))\n",
    "print(v_cache[0][0].shape)\n",
    "print(v_cache[0][-1].shape)\n",
    "print(len(v_cache))\n",
    "print(len(v_cache[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# we now have the final embedding, the best guess the model could make about the next token\n",
    "the shape of the embedding is the same as regular token embeddings [7x2560] where 7 is the number of tokens and 2560 is the embedding dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4096])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_embedding = rms_norm(final_embedding, model[\"transformer.encoder.final_layernorm.weight\"])\n",
    "final_embedding.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# finally, lets decode the embedding into the token value\n",
    "we will use the output decoder to convert the final embedding into a token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([65024, 4096])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model[\"transformer.output_layer.weight\"].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# we use the embedding of the last token to predict the next value\n",
    "hopefully in our case: ninety "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([65024])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = torch.matmul(final_embedding[-1], model[\"transformer.output_layer.weight\"].T.float())\n",
    "#logits = torch.matmul(final_embedding, model[\"lm_head.weight\"].T)\n",
    "logits.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the model predicted token number 77876 as the next token, is this the token number for ninety?\n",
    "IM HYPING YOU UP, this is the last cell of code, hopefully you had fun :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(23862)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next_token = torch.argmax(logits, dim=-1)\n",
    "next_token"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lets fucking go"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'nin'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp_model.decode([next_token.item()])\n",
    "#tokenizer.decode(list(next_token.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prefill stage is finish, we get the k_cache and v_cache for input_token.\n",
    "we also get a output(next token), and we can use it as a input token for generat next token. This is the feature for Decoder only model.\n",
    "##### 预填充阶段推理完成，并生成第一个token。同时缓存了prompt阶段过程中生成的k和v，用于下一阶段进行decoder推理，进行自回归生成，同时加载前面生成的k和v，进行加速推理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_new_len = 20\n",
    "seq_len = len(tokens)\n",
    "GQA_num = n_heads // n_kv_heads \n",
    "GQA_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23862\n",
      "token decode:  \n",
      "\n",
      "13\n",
      "token decode:  Gen\n",
      "15337\n",
      "token decode:  ius\n",
      "3551\n",
      "token decode:  is\n",
      "323\n",
      "token decode:  one\n",
      "544\n",
      "token decode:  percent\n",
      "2098\n",
      "token decode:  inspiration\n",
      "8969\n",
      "token decode:  and\n",
      "293\n"
     ]
    }
   ],
   "source": [
    "next_token = torch.tensor([next_token.item()])\n",
    "for _ in range(max_new_len-1):\n",
    "    print(next_token[-1].item())\n",
    "    if next_token[-1].item() == 2:    # 2 is \"eos_id\"\n",
    "        break\n",
    "    next_token = next_token[-1:]\n",
    "    next_token_embeddings_unnormalized = embedding_layer(next_token).to(torch.bfloat16)\n",
    "\n",
    "    final_embedding = next_token_embeddings_unnormalized\n",
    "    for layer in range(n_layers):\n",
    "        qkv_attention_store = []\n",
    "        layer_embedding_norm = rms_norm(final_embedding, model[f\"transformer.encoder.layers.{layer}.input_layernorm.weight\"])\n",
    "        q_k_v_layer = model[f\"transformer.encoder.layers.{layer}.self_attention.query_key_value.weight\"]\n",
    "        head_dim = q_k_v_layer.shape[1] // n_heads\n",
    "        q_layer, k_layer, v_layer = q_k_v_layer.split([ \n",
    "                                                        n_heads * n_kv_channels,\n",
    "                                                        n_kv_heads * n_kv_channels,\n",
    "                                                        n_kv_heads * n_kv_channels,\n",
    "                                                      ], dim=0)\n",
    "        q_layer = q_layer.reshape(n_heads, n_kv_channels, -1)\n",
    "\n",
    "        q_k_v_layer_bias = model[f\"transformer.encoder.layers.{layer}.self_attention.query_key_value.bias\"]\n",
    "        q_layer_bias, k_layer_bias, v_layer_bias = q_k_v_layer_bias.split([\n",
    "                                                                            n_heads * n_kv_channels,\n",
    "                                                                            n_kv_heads * n_kv_channels,\n",
    "                                                                            n_kv_heads * n_kv_channels,\n",
    "                                                                          ])\n",
    "        q_layer_bias = q_layer_bias.reshape(n_heads, -1)\n",
    "        \n",
    "        k_layer = k_layer.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "        k_layer_bias = k_layer_bias.reshape(n_kv_heads, -1)\n",
    "        \n",
    "        v_layer = v_layer.reshape(n_kv_heads, n_kv_channels, -1)\n",
    "        v_layer_bias = v_layer_bias.reshape(n_kv_heads, -1)\n",
    "        \n",
    "        for head in range(n_heads):\n",
    "            q_layer_head = q_layer[head]\n",
    "            k_layer_head = k_layer[head//GQA_num]\n",
    "            v_layer_head = v_layer[head//GQA_num]\n",
    "            q_layer_bias_head = q_layer_bias[head]\n",
    "            k_layer_bias_head = k_layer_bias[head//GQA_num]\n",
    "            v_layer_bias_head = v_layer_bias[head//GQA_num]\n",
    "            q_per_token = torch.matmul(layer_embedding_norm, q_layer_head.T.float()) + q_layer_bias_head.float()\n",
    "            q_per_token_split_into_pairs = q_per_token.float().view(q_per_token.shape[0], -1, 2)\n",
    "            q_per_token_as_complex_numbers = torch.view_as_complex(q_per_token_split_into_pairs)\n",
    "            freqs_for_next_token = torch.outer(torch.tensor([seq_len]), freqs)\n",
    "            freqs_cis_next_token = torch.polar(torch.ones_like(freqs_for_next_token), freqs_for_next_token)\n",
    "            q_per_token_split_into_pairs_rotated = torch.view_as_real(q_per_token_as_complex_numbers * freqs_cis_next_token)\n",
    "            q_per_token_rotated = q_per_token_split_into_pairs_rotated.view(q_per_token.shape)\n",
    "\n",
    "            if head % GQA_num == 0:\n",
    "                v_per_token = torch.matmul(layer_embedding_norm, v_layer_head.T.float()) + v_layer_bias_head.float()\n",
    "                v_cache[layer][head//GQA_num] = torch.cat([v_cache[layer][head//GQA_num], v_per_token], dim=0)    # update v_cache\n",
    "\n",
    "                k_per_token = torch.matmul(layer_embedding_norm, k_layer_head.T.float()) + k_layer_bias_head.float()\n",
    "                k_per_token_split_into_pairs = k_per_token.float().view(k_per_token.shape[0], -1, 2)\n",
    "                k_per_token_as_complex_numbers = torch.view_as_complex(k_per_token_split_into_pairs)\n",
    "                k_per_token_split_into_pairs_rotated = torch.view_as_real(k_per_token_as_complex_numbers * freqs_cis_next_token)\n",
    "                k_per_token_rotated = k_per_token_split_into_pairs_rotated.view(k_per_token.shape)\n",
    "                k_cache[layer][head//GQA_num] = torch.cat([k_cache[layer][head//GQA_num], k_per_token_rotated], dim=0)    # update k_cache\n",
    "                \n",
    "            qk_per_token = torch.matmul(q_per_token_rotated, k_cache[layer][head//GQA_num].T)/(128)**0.5\n",
    "            #mask = torch.full((len(token_embeddings_unnormalized), len(token_embeddings_unnormalized)), float(\"-inf\"))   # don't need mask\n",
    "            #mask = torch.triu(mask, diagonal=1)\n",
    "            qk_per_token_after_masking = qk_per_token  # + mask\n",
    "            qk_per_token_after_masking_after_softmax = torch.nn.functional.softmax(qk_per_token_after_masking, dim=1)  #.to(torch.bfloat16)\n",
    "            qkv_attention = torch.matmul(qk_per_token_after_masking_after_softmax, v_cache[layer][head//GQA_num])\n",
    "            qkv_attention_store.append(qkv_attention)\n",
    "\n",
    "        stacked_qkv_attention = torch.cat(qkv_attention_store, dim=-1)\n",
    "        w_layer = model[f\"transformer.encoder.layers.{layer}.self_attention.dense.weight\"]\n",
    "        embedding_delta = torch.matmul(stacked_qkv_attention, w_layer.T.float())\n",
    "        embedding_after_edit = final_embedding + embedding_delta\n",
    "        embedding_after_edit_normalized = rms_norm(embedding_after_edit, model[f\"transformer.encoder.layers.{layer}.post_attention_layernorm.weight\"])\n",
    "        w1 = model[f\"transformer.encoder.layers.{layer}.mlp.dense_h_to_4h.weight\"]\n",
    "        w2 = model[f\"transformer.encoder.layers.{layer}.mlp.dense_4h_to_h.weight\"]\n",
    "        x = torch.matmul(embedding_after_edit_normalized, w1.T.float())\n",
    "        x = torch.chunk(x, 2, dim=-1)\n",
    "        output_after_feedforward = torch.matmul(torch.functional.F.silu(x[0]) * x[1], w2.T.float())\n",
    "        final_embedding = embedding_after_edit+output_after_feedforward\n",
    "    \n",
    "    final_embedding = rms_norm(final_embedding, model[\"transformer.encoder.final_layernorm.weight\"])\n",
    "    #print(\"final_embedding.shape: \", final_embedding.shape)\n",
    "    logits = torch.matmul(final_embedding, model[\"transformer.output_layer.weight\"].T.float())\n",
    "    #print(\"logits.shape: \", logits.shape)\n",
    "    next_token = torch.argmax(logits, dim=-1)\n",
    "    #print(\"next_token: \", next_token)\n",
    "    print(\"token decode: \", sp_model.decode([next_token.item()]))\n",
    "    seq_len += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# thank you, i love you :)\n",
    "\n",
    "This is the end. Hopefully you enjoyed reading it!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
